7.3: fix nested variables, catch-alls, paths and shared configs

- %Name% lookups also resolve inside the values they bring in, and unknown
  names are left as they are instead of crashing.
- Catch-all patterns ("*") are a fallback: specific patterns are tried first
  across all outputs, so a catch-all output listed first no longer takes
  every file.
- Directory paths only get %Name% lookups and date/time variables, so
  ! & @ ~ ^ in a path stay as they are; a leading ~ is the home directory.
- A directory config missing keys (one shared by a directory's input and
  output roles) gets them from the template instead of crashing SaraSortd.
- The history of a directory that sorts into itself compares file names as
  they are, so names with variable characters aren't re-sorted forever.

Co-Authored-By: Claude Opus 5.5 <[email protected]>

GlobalConf.toml

@@ -1,5 +1,5 @@
 Title = "SaraSortd Config"
-Version = "7.2"
+Version = "7.3"
 SafeMode = 1
 SilentMode = 0
 SortTimer = 1

InputDirConf.toml

@@ -1,5 +1,5 @@
 Title = "Unset"
-Version = "7.2"
+Version = "7.3"
 ParentDir = "Unset"
 SortDirs = 0
 

OutputDirConf.toml

@@ -1,5 +1,5 @@
 Title = "Unset"
-Version = "7.2"
+Version = "7.3"
 LastFile = "Unset"
 ParentDir = "Unset"
 ValidInputDirs = ["*"]

README.md

@@ -1,6 +1,6 @@
 <img align="center" src="https://git.srhtnin.garden/sarah/RepoAssets/raw/branch/main/SaraSortd/Banners/SaraSortd.png" width="100%" alt="SaraSortd: a file and directory sorting daemon built with python."/>
 
-(Version = "7.2")
+(Version = "7.3")
 
 
 It's made by Sarah, hence the name. :3
@@ -58,7 +58,7 @@ Here's a step-by-step tutorial (in case you don't want to watch the video).
 ```
 Title = "SaraSortd Config" # NOTE: Do not change the title. This is so that the program knows which config file is which.
 
-Version = "7.2" # Just the version number for the config file.
+Version = "7.3" # Just the version number for the config file.
 
 SafeMode = 1 # If SaraSortd should stop if an error with the config appears. 1 is stop, 0 is continue.
 
@@ -73,6 +73,7 @@ Logging = 1 # If SaraSortd's logging system should be active. If this is 0, noth
 OverwriteDirConf = 0 # If SaraSortd should overwrite directory configs with the config templates. 0 is no, 1 is yes.
 
 [Variables] # Only use these in config files, not the actual file names themselves.
+# Directory paths (InputDir, OutputDir, LogDir, FailedDir, ValidInputDirs, DirConfDir, [DirConfs]) only use %Name% lookups and the date and time variables (since 7.3). The single-character variables below are about the file being sorted, so a path like "%RootDir%/Stuff&Things" stays as it is. A "~" at the start of a path is your home directory.
 NextNum = "!" # The next available number. If 1 already exists, 2 will be used.
 
 NextChar = "&" # The next available character. If A already exists, B will be used.
@@ -103,6 +104,7 @@ RootDir = "Unset" # A variable directory. This doesn't NEED to exist for the pro
 InputDir = ["%RootDir%"] # If files get put here, they'll be sorted. This is a list, so you can add multiple. E.G: ["%RootDir%", "%RootDir%/input"]
 
 OutputDir = ["%RootDir%/Output"] # Where sorted files go. This is a list, so you can add multiple. E.G: ["%RootDir%/Output", "%RootDir%/Output2"]. The output directories have separate config files.
+# Outputs are tried in this order, first with their specific patterns. Catch-all patterns ("*") are only a fallback for files no specific pattern took (since 7.3), so an output that takes everything can be anywhere in the list.
 
 LogDir = "%RootDir%/.Logs" # Where log files from SaraSortd go.
 
@@ -120,7 +122,7 @@ LogFileName = "[y].[m].[d]" # Name for log files.
 
 InDirConfName = ".@Config" # Name for config files for input directories.
 
-OutDirConfName = ".@Config" # Name for config files for output directories. If a directory is both an input and an output and both names are the same, it uses one config file for both.
+OutDirConfName = ".@Config" # Name for config files for output directories. If a directory is both an input and an output and both names are the same, it uses one config file for both: its [[Files]] are then both the input's and the output's rules, and SaraSortd gives the file every key both roles need (since 7.3).
 
 FileHistoryName = ".@History" # Name for file history files. This keeps track of sorted files if an input and output are the same directory.
 
@@ -167,7 +169,7 @@ The output directory config references the SaraSortd config (GlobalConf.toml). T
 ```
 Title = "Unset" # NOTE: Do not change the title. This is so that the program knows which config file is which.
 
-Version = "7.2" # The version number of the config.
+Version = "7.3" # The version number of the config.
 
 LastFile = "Unset" # The last file that was sorted to this directory. It gets automatically updated whenever a file gets sorted to this directory.
 
@@ -223,7 +225,7 @@ Overwrite = 1 # Whether this specific file type can be overwritten by others of
 ```
 Title = "Unset" # NOTE: Do not change the title. This is so that the program knows which config file is which.
 
-Version = "7.2"
+Version = "7.3"
 
 ParentDir = "Unset" # The name of this directory. It gets automatically updated when this config gets copied to an output directory.
 

SaraSortd.py

@@ -6,7 +6,7 @@ import shutil
 import time
 import toml
 
-# Version = "7.2"
+# Version = "7.3"
 
 ConfPath = "./GlobalConf.toml"
 
@@ -151,8 +151,10 @@ def Parse(
                 return ""
 
         Value = GetConf(Content, Path)
-        if Value is not None:
+        if Value is None:
-            return Value
+            return Match.group(0)  # Unknown names are left as they are (they used to crash).
+
+        return ResolveLookups(str(Value), Path)
 
     Now = datetime.datetime.now()
     if OrgFile is not None:
@@ -189,9 +191,45 @@ def Parse(
     return String
 
 
+def ResolveLookups(String, Path = ConfPath, Depth = 0):
+    # %Name% lookups, also inside the values they bring in (since 7.3).
+    # E.G: ImageDir = "%RootDir%/Images" used as "%ImageDir%" gives the full path. Unknown names are left as they are.
+    def Replacer(Match):
+        Value = GetConf(Match.group(1), Path) if Match.group(1) else None
+        if Value is None or Depth >= 10:
+            return Match.group(0)
+
+        return ResolveLookups(str(Value), Path, Depth + 1)
+
+    VarCall0 = re.escape(ConfVars["VarCall"][0])
+    VarCall1 = re.escape(ConfVars["VarCall"][1])
+    return re.sub(f"{VarCall0}(.*?){VarCall1}", Replacer, String)
+
+
+def ParsePath(String):
+    # Directory paths only get %Name% lookups and the date and time variables (since 7.3). The single-character
+    # variables (NextNum, NextChar, Parent, OrgFileName, OrgFileType) are about the file being sorted, so a path
+    # like "~/Stuff&Things" stays as it is. A "~" at the start is your home directory.
+    Now = datetime.datetime.now()
+    DateValues = {
+        ConfVars["Year"]: str(Now.year),
+        ConfVars["Month"]: str(Now.month).rjust(2, "0"),
+        ConfVars["Day"]: str(Now.day).rjust(2, "0"),
+        ConfVars["Hour"]: str(Now.hour).rjust(2, "0"),
+        ConfVars["Minute"]: str(Now.minute).rjust(2, "0"),
+        ConfVars["Second"]: str(Now.second).rjust(2, "0"),
+    }
+
+    for Key, Value in DateValues.items():
+        if Key and Key in String:
+            String = String.replace(Key, Value)
+
+    return os.path.expanduser(ResolveLookups(String))
+
+
 def ConfFolder(PathText):
     # Relative paths in GlobalConf.toml (DirConfDir, [DirConfs]) count from GlobalConf.toml's own folder.
-    PathText = Parse(String = PathText)
+    PathText = ParsePath(PathText)
     if os.path.isabs(PathText):
         return os.path.normpath(PathText)
     return os.path.normpath(os.path.join(os.path.dirname(os.path.abspath(ConfPath)), PathText))
@@ -200,7 +238,7 @@ def ConfFolder(PathText):
 def FlatName(Path):
     # A directory's name for files kept outside it: its path below RootDir (or its whole path), "/" turned into "_".
     # E.G: "%RootDir%/Pictures/Cats" becomes "Pictures_Cats".
-    Root = Parse(String = ConfDirs.get("RootDir", "")).rstrip("/")
+    Root = ParsePath(ConfDirs.get("RootDir", "")).rstrip("/")
     Path = Path.rstrip("/")
     if Root and Path.startswith(Root + "/"):
         Relative = Path[len(Root) + 1:]
@@ -214,7 +252,7 @@ def CentralConf(Path, Kind):
     # [DirConfs] names a file per directory ("dir" = "file", or "dir" = { In = "file", Out = "file" }),
     # and DirConfDir holds the configs of every directory [DirConfs] doesn't name. None means inside the directory.
     for Key, Value in (Conf.get("DirConfs") or {}).items():
-        if Parse(String = Key).rstrip("/") == Path.rstrip("/"):
+        if ParsePath(Key).rstrip("/") == Path.rstrip("/"):
             if isinstance(Value, dict):
                 Value = Value.get(Kind)
             if Value:
@@ -275,6 +313,32 @@ def Clone(Source, Destination, NewName, Delete = False):
         os.remove(Source)
 
 
+def CompleteDirConf(DirConf, Template, Kind):
+    # Gives an existing directory config the keys it's missing from the template (since 7.3). A directory that is
+    # both an input and an output shares one config by default, so the second role's keys used to be missing and
+    # SaraSortd crashed. Output configs' [[Files]] entries also get the output keys (NewFileName, NextNum, ...).
+    Data = toml.load(DirConf)
+    TemplateData = toml.load(Template)
+    Changed = False
+
+    for Key, Value in TemplateData.items():
+        if Key != "Files" and Key not in Data:
+            Data[Key] = Value
+            Changed = True
+
+    if Kind == "Out":
+        Defaults = (TemplateData.get("Files") or [{}])[0]
+        for Entry in Data.get("Files", []):
+            for Key, Value in Defaults.items():
+                if Key not in Entry:
+                    Entry[Key] = Value
+                    Changed = True
+
+    if Changed:
+        with open(DirConf, "w", encoding = "utf-8") as File:
+            toml.dump(Data, File)
+
+
 def Dir(Path, Output = True, CopyConf = "OutputDirConf.toml"):
     ParentName = os.path.basename(Path)
     os.makedirs(Path, exist_ok=True)
@@ -291,9 +355,12 @@ def Dir(Path, Output = True, CopyConf = "OutputDirConf.toml"):
                 UpdateConf(DirConf, "ParentDir", ParentName)
                 UpdateConf(DirConf, "Title", f"{ParentName} Config")
 
+            else:
+                CompleteDirConf(DirConf, CopyConf, Kind)
+
     if Output:
-        ParsedPath = Parse(String = Path)
+        ParsedPath = ParsePath(Path)
-        ParsedOutputs = [Parse(String = Dir) for Dir in ConfDirs["OutputDir"]]
+        ParsedOutputs = [ParsePath(Dir) for Dir in ConfDirs["OutputDir"]]
         if ParsedPath not in ParsedOutputs:
             UpdateConf(ConfPath, "OutputDir", Path, True)
 
@@ -305,8 +372,8 @@ def LogWrite(Text):
                 ConfNames["LogFileName"] is not None
                 and ConfNames["LogFileName"] != "Unset"
             ):
-                Dir(Parse(String = ConfDirs["LogDir"], Path = ConfPath), Output = False, CopyConf = False)
+                Dir(ParsePath(ConfDirs["LogDir"]), Output = False, CopyConf = False)
-                LogFile = f"{Parse(String = ConfDirs['LogDir'], Path = ConfPath)}/{Parse(String = ConfNames['LogFileName'], Path = ConfPath)}.log"
+                LogFile = f"{ParsePath(ConfDirs['LogDir'])}/{Parse(String = ConfNames['LogFileName'], Path = ConfPath)}.log"
                 with open(LogFile, "a", encoding = "utf-8", buffering = 1) as File:
                     File.write(f"{Parse(String = ConfLog['All'], Path = ConfPath)}{Text}\n")
 
@@ -320,17 +387,23 @@ def FileInputValid(File, FilePath, ParentName):
         return True
 
     for ValidInputDir in ValidInputDirs:
-        if fnmatch.fnmatchcase(os.path.dirname(FilePath), Parse(String = ValidInputDir, Parent = ParentName)):
+        if fnmatch.fnmatchcase(os.path.dirname(FilePath), ParsePath(ValidInputDir)):
             return True
 
     return False
 
 
+def IsCatchAll(Pattern):
+    # A pattern made of only "*" takes any file.
+    return Pattern != "" and set(Pattern) == {"*"}
+
+
 def DecideNewPath(FilePath):
     UnsortedFile = os.path.basename(FilePath)
+    Candidates = []
 
     for Output in ConfDirs["OutputDir"]:
-        Output = Parse(String = Output, Path = ConfPath)
+        Output = ParsePath(Output)
 
         ParentName = os.path.basename(Output)
 
@@ -340,7 +413,7 @@ def DecideNewPath(FilePath):
         InputDirValid = False
 
         for ValidInputDir in ValidInputDirs:
-            ValidInputDir = Parse(String = ValidInputDir, Parent = GetConf("ParentDir", OutDirConfPath))
+            ValidInputDir = ParsePath(ValidInputDir)
 
             TextOutput = Parse(String = ConfLog["MatchPattern"], VarCall = f"{os.path.dirname(FilePath)} against {ValidInputDir}")
             LogWrite(TextOutput)
@@ -372,88 +445,94 @@ def DecideNewPath(FilePath):
         if not FileConf:
             continue
 
-        for FileIndex, File in enumerate(FileConf):
+        Candidates.append((Output, ParentName, OutDirConfPath, FileConf))
-            if not FileInputValid(File, FilePath, ParentName):
-                continue
 
-            Pattern = File["Pattern"]
+    # Specific patterns first, in OutputDir order. Catch-alls ("*") are a fallback for files nothing else took,
-
+    # so an output with the template's catch-all doesn't take every file just because it's listed first (since 7.3).
-            TextOutput = Parse(String = ConfLog["MatchPattern"], VarCall = f"{UnsortedFile} against {Pattern}")
+    for CatchAll in (False, True):
-            LogWrite(TextOutput)
+        for Output, ParentName, OutDirConfPath, FileConf in Candidates:
-            Speak(TextOutput)
+            for FileIndex, File in enumerate(FileConf):
-
+                if IsCatchAll(File["Pattern"]) != CatchAll or not FileInputValid(File, FilePath, ParentName):
-            if File["CaseSensitive"] == 1:
-                Match = fnmatch.fnmatchcase(UnsortedFile, Pattern)
-
-            else:
-                Match = fnmatch.fnmatch(UnsortedFile.lower(), Pattern.lower())
-
-            if Match:
-                NewFileName = File["NewFileName"]
-
-                NewFileName = Parse(String = NewFileName, Path = OutDirConfPath, OrgFile = UnsortedFile, NextNum = File["NextNum"], NextChar = File["NextChar"], Parent = ParentName)
-
-                if File["Overwrite"] == 0 and os.path.exists(os.path.join(Output, NewFileName)):
                     continue
 
-                if ConfVars["NextNum"] in File["NewFileName"]:
+                Pattern = File["Pattern"]
-                    ConfData = toml.load(OutDirConfPath)
-                    Width = len(File["NextNum"])
-                    NewNum = str((int(File["NextNum"]) + 1)).rjust(Width, "0")
-                    print(File["NextNum"],"=>",NewNum)
 
-                    # By position: two entries can share a Pattern (with different ValidInputDirs).
+                TextOutput = Parse(String = ConfLog["MatchPattern"], VarCall = f"{UnsortedFile} against {Pattern}")
-                    ConfData["Files"][FileIndex]["NextNum"] = NewNum
+                LogWrite(TextOutput)
+                Speak(TextOutput)
 
-                    with open(OutDirConfPath, "w", encoding="utf-8") as ConfFile:
+                if File["CaseSensitive"] == 1:
-                        toml.dump(ConfData, ConfFile)
+                    Match = fnmatch.fnmatchcase(UnsortedFile, Pattern)
 
-                    TextOutput = Parse(String = ConfLog["ValueSet"], VarCall = f"NextNum to {NewNum}")
+                else:
-                    LogWrite(TextOutput)
+                    Match = fnmatch.fnmatch(UnsortedFile.lower(), Pattern.lower())
-                    Speak(TextOutput)
 
-                if ConfVars["NextChar"] in File["NewFileName"]:
+                if Match:
-                    CurrentChar = File["NextChar"]
+                    NewFileName = File["NewFileName"]
-                    CurrentChar = CurrentChar.upper()
-                    Result = list(CurrentChar)
 
-                    Count = len(Result) - 1
+                    NewFileName = Parse(String = NewFileName, Path = OutDirConfPath, OrgFile = UnsortedFile, NextNum = File["NextNum"], NextChar = File["NextChar"], Parent = ParentName)
 
-                    while Count >= 0:
+                    if File["Overwrite"] == 0 and os.path.exists(os.path.join(Output, NewFileName)):
-                        if Result[Count] == "Z":
+                        continue
-                            Result[Count] = "A"
+
-                            Count -= 1
+                    if ConfVars["NextNum"] in File["NewFileName"]:
+                        ConfData = toml.load(OutDirConfPath)
+                        Width = len(File["NextNum"])
+                        NewNum = str((int(File["NextNum"]) + 1)).rjust(Width, "0")
+                        print(File["NextNum"],"=>",NewNum)
+
+                        # By position: two entries can share a Pattern (with different ValidInputDirs).
+                        ConfData["Files"][FileIndex]["NextNum"] = NewNum
+
+                        with open(OutDirConfPath, "w", encoding="utf-8") as ConfFile:
+                            toml.dump(ConfData, ConfFile)
+
+                        TextOutput = Parse(String = ConfLog["ValueSet"], VarCall = f"NextNum to {NewNum}")
+                        LogWrite(TextOutput)
+                        Speak(TextOutput)
+
+                    if ConfVars["NextChar"] in File["NewFileName"]:
+                        CurrentChar = File["NextChar"]
+                        CurrentChar = CurrentChar.upper()
+                        Result = list(CurrentChar)
+
+                        Count = len(Result) - 1
+
+                        while Count >= 0:
+                            if Result[Count] == "Z":
+                                Result[Count] = "A"
+                                Count -= 1
+
+                            else:
+                                Result[Count] = chr(ord(Result[Count]) + 1)
+                                break
 
                         else:
-                            Result[Count] = chr(ord(Result[Count]) + 1)
+                            Result.insert(0, "A")
-                            break
 
-                    else:
+                        NewChar = "".join(Result)
-                        Result.insert(0, "A")
 
-                    NewChar = "".join(Result)
+                        ConfData = toml.load(OutDirConfPath)
 
-                    ConfData = toml.load(OutDirConfPath)
+                        ConfData["Files"][FileIndex]["NextChar"] = NewChar
 
-                    ConfData["Files"][FileIndex]["NextChar"] = NewChar
+                        with open(OutDirConfPath, "w", encoding="utf-8") as ConfFile:
+                            toml.dump(ConfData, ConfFile)
 
-                    with open(OutDirConfPath, "w", encoding="utf-8") as ConfFile:
+                        TextOutput = Parse(String = ConfLog["ValueSet"], VarCall = f"NextChar to {NewChar}")
-                        toml.dump(ConfData, ConfFile)
+                        LogWrite(TextOutput)
+                        Speak(TextOutput)
 
-                    TextOutput = Parse(String = ConfLog["ValueSet"], VarCall = f"NextChar to {NewChar}")
+                    return f"{Output}/{NewFileName}"
-                    LogWrite(TextOutput)
-                    Speak(TextOutput)
-
-                return f"{Output}/{NewFileName}"
 
 
 def Sort(FilePath):
     NewPath = DecideNewPath(FilePath)
 
     if NewPath is None:
-        Dir(Parse(String = ConfDirs["FailedDir"]), Output = False, CopyConf = False)
+        Dir(ParsePath(ConfDirs["FailedDir"]), Output = False, CopyConf = False)
         NewName = os.path.basename(FilePath)
-        Clone(FilePath, Parse(String = ConfDirs["FailedDir"]), NewName, True)
+        Clone(FilePath, ParsePath(ConfDirs["FailedDir"]), NewName, True)
 
         TextOutput = Parse(String = ConfLog["NotSorted"], VarCall = FilePath)
         LogWrite(TextOutput)
@@ -478,9 +557,9 @@ def Sort(FilePath):
         Speak(TextOutput)
 
     except Exception:
-        Dir(Parse(String = ConfDirs["FailedDir"]), Output = False, CopyConf = False)
+        Dir(ParsePath(ConfDirs["FailedDir"]), Output = False, CopyConf = False)
         NewName = os.path.basename(NewPath)
-        Clone(FilePath, Parse(String = ConfDirs["FailedDir"]), NewName, Delete)
+        Clone(FilePath, ParsePath(ConfDirs["FailedDir"]), NewName, Delete)
 
         TextOutput = Parse(String = ConfLog["NotSorted"], VarCall = FilePath)
         LogWrite(TextOutput)
@@ -497,11 +576,11 @@ def Init():
 
         if Start:
             for InputDir in ConfDirs["InputDir"]:
-                InputDir = Parse(String = InputDir, Path = ConfPath, Parent = os.path.basename(os.path.dirname(InputDir)))
+                InputDir = ParsePath(InputDir)
                 Dir(InputDir, Output = False, CopyConf = "InputDirConf.toml")
 
             for Output in ConfDirs["OutputDir"]:
-                Output = Parse(String = Output, Parent = os.path.basename(os.path.dirname(Output)))
+                Output = ParsePath(Output)
                 Dir(Output)
 
                 OutDirConfPath = DirConfPath(Output, "Out")
@@ -538,7 +617,7 @@ def Main():
             for Input in ConfDirs["InputDir"]:
                 RecurseDir = False
 
-                InputPath = Parse(Input)
+                InputPath = ParsePath(Input)
 
                 if not os.path.exists(InputPath):
                     continue
@@ -547,10 +626,10 @@ def Main():
                 InputDirConf = toml.load(InputDirConfPath)
 
                 for Output in ConfDirs["OutputDir"]:
-                    OutputPath = Parse(Output)
+                    OutputPath = ParsePath(Output)
                     OutDirConfPath = DirConfPath(OutputPath, "Out")
 
-                    if len(GetConf("Unzip", OutDirConfPath)) >= 1:
+                    if len(GetConf("Unzip", OutDirConfPath) or []) >= 1:
                         ZipTypes = GetConf("Unzip", OutDirConfPath)
                         for OutputFile in os.listdir(OutputPath):
                             if OutputFile.startswith("."):
@@ -583,7 +662,7 @@ def Main():
 
                 for FileName in os.listdir(InputPath):
                     FilePath = f"{InputPath}/{FileName}"
-                    if os.path.isdir(FilePath) and InputDirConf["SortDirs"] == 0:
+                    if os.path.isdir(FilePath) and InputDirConf.get("SortDirs", 0) == 0:
                         continue
 
                     FileName = os.path.basename(FilePath)
@@ -601,7 +680,7 @@ def Main():
                             break
 
                     if Match:
-                        if os.path.isdir(FilePath) and InputDirConf["SortDirs"] == 1:
+                        if os.path.isdir(FilePath) and InputDirConf.get("SortDirs", 0) == 1:
                             TextOutput = Parse(String = ConfLog["Zipping"], VarCall = FilePath)
                             LogWrite(TextOutput)
                             Speak(TextOutput)
@@ -622,7 +701,7 @@ def Main():
                         if RecurseDir:
                             HistoryFile = toml.load(FileHistoryPath)
 
-                            if (Parse(FileName, Parent = os.path.basename(InputPath)) in HistoryFile["History"]):
+                            if (FileName in HistoryFile["History"]):
                                 continue
 
                         Sort(FilePath)