Variables: one engine for server and client, and spans for live editors

Co-Authored-By: Claude Opus 5.5 <[email protected]>

Backend/VarEngine.py

@@ -0,0 +1,482 @@
+"""SaraView's variables (portals, definitions, loops), resolved: the one
+engine both the server (VarFunctions.py) and the desktop client (local
+folders) use, so a note reads the same everywhere.
+
+It never touches files itself: an Access object reads them, so the server's
+can check locks and the client's can read a folder on disk. Paths are
+relative to the root, like the server's.
+
+ResolvePortals gives the note with everything filled in (the rendered view);
+ResolveSpans gives where each variable is in the note and what it becomes,
+for the live editors to show results in place of the code."""
+
+from __future__ import annotations
+
+import re
+from typing import Optional, Protocol
+
+NoteFilePrefix = ".sv_"
+
+
+class Access(Protocol):
+    def ReadText(self, RelativePath: str) -> Optional[str]:
+        """A Markdown or text file's text, or None (missing, another type, or not allowed)."""
+
+    def List(self, DirectoryPath: str) -> Optional[list[dict]]:
+        """The entries ({"Name", "IsDirectory", "FileType"}) you're allowed to
+        see in a directory, or None if it isn't one or you can't."""
+
+    def IsDirectory(self, RelativePath: str) -> bool: ...
+
+
+def NormalizeRelativePath(BaseDirectory: str, RawPath: str) -> str:
+    Combined = f"{BaseDirectory}/{RawPath}" if BaseDirectory else RawPath
+    Parts: list[str] = []
+    for Part in Combined.split("/"):
+        if Part in ("", "."):
+            continue
+        if Part == "..":
+            if Parts:
+                Parts.pop()
+            continue
+        Parts.append(Part)
+    return "/".join(Parts)
+
+
+PortalFileTypes = {"markdown", "text"}
+LoopSources = ("children", "siblings", "parents")
+
+MarkerPattern = re.compile(
+    r"\{(?:(?P<CrossPath>[^\s{}']+)\s+)?\*\*\*(?P<CrossName>[^\s*{}]+)"
+    r"(?:\s+(?P<CrossPipe>head|tail)\s+(?P<CrossPipeCount>\d+))?"
+    r"(?:\s*\\\s*(?P<CrossClip>[^{}]*))?\}"
+    r"|\{(?P<PipePath>[^\s{}']+)\s+(?P<PipeMode>head|tail)\s+(?P<PipeCount>\d+)"
+    r"(?:\s*\\\s*(?P<PipeClip>[^{}]*))?\}"
+    r"|\{(?P<LinePath>[^\s{}']+)\s+'(?P<LineNumber>\d+)'"
+    r"(?:\s*\\\s*(?P<LineClip>[^{}]*))?\}"
+    r"|\{\*\*(?P<LineName>[^\s*{}]+)\}"
+    r"|\{\*(?P<BlockName>[^\s*{}]+)\}"
+)
+
+LoopMarkerPattern = re.compile(
+    r"\{(?P<LoopSource>children|siblings|parents)\s+%\s+(?P<LoopVar>[A-Za-z_][A-Za-z0-9_]*)\}"
+    r"|\{(?P<CloseSource>children|siblings|parents)\s+%\}"
+    r"|\{(?:(?P<LoopCrossPath>[^\s{}']+)\s+)?\*\*\*(?P<LoopCrossName>[^\s*{}]+)"
+    r"(?:\s+(?P<LoopCrossPipe>head|tail)\s+(?P<LoopCrossPipeCount>\d+))?"
+    r"\s+%\s+(?P<LoopCrossVar>[A-Za-z_][A-Za-z0-9_]*)\}"
+    r"|\{\*\*\*(?P<CloseCrossName>[^\s*{}]+)\s+%\}"
+)
+
+
+def _CompileClipPattern(Pattern: str) -> re.Pattern:
+    RegexParts = []
+    LastIndex = len(Pattern) - 1
+
+    for Index, Char in enumerate(Pattern):
+        IsLast = Index == LastIndex
+        if Char == "!":
+            RegexParts.append("(.*)" if IsLast else "(.*?)")
+        elif Char == "?":
+            RegexParts.append(".*" if IsLast else ".*?")
+        else:
+            RegexParts.append(re.escape(Char))
+
+    if not Pattern or Pattern[-1] not in "!?":
+        RegexParts.append(".*")
+
+    return re.compile("".join(RegexParts), re.DOTALL)
+
+
+def _ApplyClip(Value: str, Pattern: str) -> str:
+    Pattern = Pattern.strip()
+
+    if not Pattern:
+        return Value
+
+    Match = _CompileClipPattern(Pattern).match(Value)
+
+    if Match is None:
+        return Value
+
+    return "".join(Match.groups())
+
+
+def _LineBounds(Content: str, Position: int) -> tuple[int, int]:
+    LineStart = Content.rfind("\n", 0, Position) + 1
+    LineEnd = Content.find("\n", Position)
+    if LineEnd == -1:
+        LineEnd = len(Content)
+    return LineStart, LineEnd
+
+
+def _FindLocalDefinitions(Content: str) -> dict[str, str]:
+    Matches = list(MarkerPattern.finditer(Content))
+
+    LineDefinitions: dict[str, str] = {}
+    BlockOccurrences: dict[str, list[re.Match]] = {}
+
+    for Match in Matches:
+        if Match.group("LineName"):
+            Name = Match.group("LineName")
+            if Name not in LineDefinitions:
+                LineStart, LineEnd = _LineBounds(Content, Match.start())
+                Line = Content[LineStart:LineEnd]
+                Captured = (
+                    Line[: Match.start() - LineStart] + Line[Match.end() - LineStart :]
+                ).strip()
+                LineDefinitions[Name] = _ResolveLocalPortals(Captured)
+        elif Match.group("BlockName"):
+            Name = Match.group("BlockName")
+            BlockOccurrences.setdefault(Name, []).append(Match)
+
+    BlockDefinitions: dict[str, str] = {}
+    for Name, Occurrences in BlockOccurrences.items():
+        Open = Occurrences[0]
+        if len(Occurrences) >= 2:
+            Close = Occurrences[1]
+            Captured = Content[Open.end() : Close.start()].strip("\n")
+        else:
+            Captured = Content[Open.end() :].lstrip("\n")
+        BlockDefinitions[Name] = _ResolveLocalPortals(Captured)
+
+    return {**LineDefinitions, **BlockDefinitions}
+
+
+def _ResolveLocalPortals(Content: str) -> str:
+    Matches = list(MarkerPattern.finditer(Content))
+    Definitions = _FindLocalDefinitions(Content)
+
+    LineSeenCount: dict[str, int] = {}
+    BlockSeenCount: dict[str, int] = {}
+
+    Output: list[str] = []
+    Cursor = 0
+
+    for Match in Matches:
+        Replacement = None
+
+        if Match.group("LineName"):
+            Name = Match.group("LineName")
+            Occurrence = LineSeenCount.get(Name, 0)
+            LineSeenCount[Name] = Occurrence + 1
+            Replacement = "" if Occurrence == 0 else Definitions.get(Name, "")
+        elif Match.group("BlockName"):
+            Name = Match.group("BlockName")
+            Occurrence = BlockSeenCount.get(Name, 0)
+            BlockSeenCount[Name] = Occurrence + 1
+            Replacement = "" if Occurrence in (0, 1) else Definitions.get(Name, "")
+
+        if Replacement is not None:
+            Output.append(Content[Cursor : Match.start()])
+            Output.append(Replacement)
+            Cursor = Match.end()
+
+    Output.append(Content[Cursor:])
+    return "".join(Output)
+
+
+def _ListEntryNames(Files: Access, DirectoryPath: str, Exclude: str | None) -> list[str]:
+    Entries = Files.List(DirectoryPath)
+    return [Entry["Name"] for Entry in Entries or [] if Entry["Name"] != Exclude]
+
+
+def _ComputeLoopLists(Files: Access, RelativePath: str) -> dict[str, list[str]]:
+    FileName = RelativePath.rsplit("/", 1)[-1] if RelativePath else ""
+    ParentDirectory = RelativePath.rsplit("/", 1)[0] if "/" in RelativePath else ""
+    ParentBaseName = ParentDirectory.rsplit("/", 1)[-1] if ParentDirectory else ""
+
+    IsDirectoryNote = FileName.startswith(f"{NoteFilePrefix}{ParentBaseName}.")
+
+    Children = _ListEntryNames(Files, ParentDirectory, FileName) if IsDirectoryNote else []
+    Siblings = _ListEntryNames(Files, ParentDirectory, FileName)
+    Parents = [Segment for Segment in ParentDirectory.split("/") if Segment]
+
+    return {"children": Children, "siblings": Siblings, "parents": Parents}
+
+
+def _FindLoopClose(Matches: list[re.Match], StartIndex: int, IsFixed: bool, Key: str) -> re.Match | None:
+    GroupName = "CloseSource" if IsFixed else "CloseCrossName"
+
+    for Candidate in Matches[StartIndex:]:
+        if Candidate.group(GroupName) == Key:
+            return Candidate
+
+    return None
+
+
+def _ResolveLoops(
+    Content: str,
+    Lists: dict[str, list[str]],
+    Files: Access,
+    RefererDirectory: str,
+) -> str:
+    Matches = list(LoopMarkerPattern.finditer(Content))
+    Output: list[str] = []
+    Cursor = 0
+    Index = 0
+
+    while Index < len(Matches):
+        Match = Matches[Index]
+        IsFixed = bool(Match.group("LoopSource"))
+        IsCross = bool(Match.group("LoopCrossName"))
+
+        if not IsFixed and not IsCross:
+            Index += 1
+            continue
+
+        if IsFixed:
+            Key = Match.group("LoopSource")
+            LoopVar = Match.group("LoopVar")
+            Items = Lists.get(Key, [])
+        else:
+            Key = Match.group("LoopCrossName")
+            LoopVar = Match.group("LoopCrossVar")
+            PortalContent = _ResolveCrossFileReference(
+                Files, RefererDirectory, Match.group("LoopCrossPath"), Key
+            )
+
+            if PortalContent is None:
+                Items = []
+            elif Match.group("LoopCrossPipe"):
+                Items = _SlicePipeLines(
+                    PortalContent, Match.group("LoopCrossPipe"), int(Match.group("LoopCrossPipeCount"))
+                )
+            else:
+                Items = PortalContent.splitlines()
+
+        CloseMatch = _FindLoopClose(Matches, Index + 1, IsFixed, Key)
+        BodyEnd = CloseMatch.start() if CloseMatch is not None else len(Content)
+        BodyTemplate = Content[Match.end() : BodyEnd]
+        EscapedVar = re.escape(LoopVar)
+        VariablePattern = re.compile(
+            r"\{%" + EscapedVar + r"\s*\\\s*(?P<Clip>[^{}]*)\}"
+            r"|%" + EscapedVar + r"\b"
+        )
+
+        RenderedItems = []
+        for Item in Items:
+            ItemRendered = VariablePattern.sub(
+                lambda VariableMatch, CurrentItem=Item: (
+                    _ApplyClip(CurrentItem, VariableMatch.group("Clip"))
+                    if VariableMatch.group("Clip") is not None
+                    else CurrentItem
+                ),
+                BodyTemplate,
+            )
+            RenderedItems.append(ItemRendered)
+
+        Rendered = "\n".join(RenderedItems)
+
+        Output.append(Content[Cursor : Match.start()])
+        Output.append(Rendered)
+        Cursor = CloseMatch.end() if CloseMatch is not None else len(Content)
+
+        if CloseMatch is not None:
+            Index = Matches.index(CloseMatch) + 1
+        else:
+            break
+
+    Output.append(Content[Cursor:])
+    return "".join(Output)
+
+
+def _FindNamedPortalInDirectory(Files: Access, DirectoryPath: str, Name: str) -> str | None:
+    for Entry in Files.List(DirectoryPath) or []:
+        if Entry["IsDirectory"] or Entry["FileType"] not in PortalFileTypes:
+            continue
+
+        EntryPath = f"{DirectoryPath}/{Entry['Name']}" if DirectoryPath else Entry["Name"]
+        Content = Files.ReadText(EntryPath)
+
+        if Content is None:
+            continue
+
+        Definitions = _FindLocalDefinitions(Content)
+        if Name in Definitions:
+            return Definitions[Name]
+
+    return None
+
+
+def _ResolveCrossFileReference(
+    Files: Access, RefererDirectory: str, PathText: str | None, Name: str
+) -> str | None:
+    if not PathText:
+        return _FindNamedPortalInDirectory(Files, RefererDirectory, Name)
+
+    TargetRelativePath = NormalizeRelativePath(RefererDirectory, PathText)
+
+    if Files.IsDirectory(TargetRelativePath):
+        return _FindNamedPortalInDirectory(Files, TargetRelativePath, Name)
+
+    Content = Files.ReadText(TargetRelativePath)
+    if Content is None:
+        return None
+
+    return _FindLocalDefinitions(Content).get(Name)
+
+
+def _SlicePipeLines(Text: str, Mode: str, Count: int) -> list[str]:
+    Lines = Text.splitlines()
+    return Lines[:Count] if Mode == "head" else (Lines[-Count:] if Count > 0 else [])
+
+
+def _ApplyPipe(Text: str, Mode: str, Count: int) -> str:
+    return "\n".join(_SlicePipeLines(Text, Mode, Count))
+
+
+def _ResolveWholeFilePipe(Files: Access, RefererDirectory: str, PathText: str, Mode: str, Count: int) -> str | None:
+    Content = Files.ReadText(NormalizeRelativePath(RefererDirectory, PathText))
+
+    if Content is None:
+        return None
+
+    return _ApplyPipe(_ResolveLocalPortals(Content), Mode, Count)
+
+
+def _ResolveLineNumberReference(Files: Access, RefererDirectory: str, PathText: str, LineNumber: int) -> str | None:
+    Content = Files.ReadText(NormalizeRelativePath(RefererDirectory, PathText))
+
+    if Content is None:
+        return None
+
+    Lines = Content.splitlines()
+    Index = LineNumber - 1
+
+    if 0 <= Index < len(Lines):
+        return Lines[Index]
+
+    return None
+
+
+def ResolvePortals(RelativePath: str, Content: str, Files: Access) -> str:
+    """The note with its variables filled in."""
+    RefererDirectory = RelativePath.rsplit("/", 1)[0] if "/" in RelativePath else ""
+    LoopLists = _ComputeLoopLists(Files, RelativePath)
+    WithLoopsResolved = _ResolveLoops(Content, LoopLists, Files, RefererDirectory)
+    LocallyResolved = _ResolveLocalPortals(WithLoopsResolved)
+
+    Matches = list(MarkerPattern.finditer(LocallyResolved))
+    Output: list[str] = []
+    Cursor = 0
+
+    for Match in Matches:
+        Replacement = None
+
+        if Match.group("CrossName"):
+            Replacement = _ResolveCrossFileReference(
+                Files, RefererDirectory, Match.group("CrossPath"), Match.group("CrossName")
+            )
+            if Replacement is not None and Match.group("CrossPipe"):
+                Replacement = _ApplyPipe(
+                    Replacement, Match.group("CrossPipe"), int(Match.group("CrossPipeCount"))
+                )
+            if Replacement is not None and Match.group("CrossClip"):
+                Replacement = _ApplyClip(Replacement, Match.group("CrossClip"))
+        elif Match.group("PipeMode"):
+            Replacement = _ResolveWholeFilePipe(
+                Files, RefererDirectory, Match.group("PipePath"), Match.group("PipeMode"), int(Match.group("PipeCount"))
+            )
+            if Replacement is not None and Match.group("PipeClip"):
+                Replacement = _ApplyClip(Replacement, Match.group("PipeClip"))
+        elif Match.group("LineNumber"):
+            Replacement = _ResolveLineNumberReference(
+                Files, RefererDirectory, Match.group("LinePath"), int(Match.group("LineNumber"))
+            )
+            if Replacement is not None and Match.group("LineClip"):
+                Replacement = _ApplyClip(Replacement, Match.group("LineClip"))
+
+        if Replacement is not None:
+            Output.append(LocallyResolved[Cursor : Match.start()])
+            Output.append(Replacement)
+            Cursor = Match.end()
+
+    Output.append(LocallyResolved[Cursor:])
+    return "".join(Output)
+
+
+# --- Where each variable is, and what it becomes (the live editors) --------
+
+_Token = re.compile("\ue000(\\d+)\ue001|\ue002(\\d+)\ue003")
+
+
+def _TopLevelVariables(Content: str) -> tuple[list[tuple[int, int]], list[tuple[int, int]]]:
+    """(Variables, Definitions). Variables: each loop as a whole (its opening
+    to its closing marker), and every other variable outside the loops.
+    Definitions: the markers that only define ({**name} the first time,
+    {*name} the first two), which always resolve to nothing."""
+    Found: list[tuple[int, int]] = []
+    Matches = list(LoopMarkerPattern.finditer(Content))
+    Index = 0
+    while Index < len(Matches):
+        Match = Matches[Index]
+        IsFixed = bool(Match.group("LoopSource"))
+        if not IsFixed and not Match.group("LoopCrossName"):
+            Index += 1
+            continue
+        Key = Match.group("LoopSource") if IsFixed else Match.group("LoopCrossName")
+        Close = _FindLoopClose(Matches, Index + 1, IsFixed, Key)
+        Found.append((Match.start(), Close.end() if Close is not None else len(Content)))
+        if Close is None:
+            break
+        Index = Matches.index(Close) + 1
+    Definitions: list[tuple[int, int]] = []
+    Seen: dict[tuple[str, str], int] = {}
+    for Match in MarkerPattern.finditer(Content):
+        if any(Start <= Match.start() < End for Start, End in Found):
+            continue
+        Kind = "Line" if Match.group("LineName") else "Block" if Match.group("BlockName") else None
+        if Kind is not None:
+            Key = (Kind, Match.group(f"{Kind}Name"))
+            Seen[Key] = Seen.get(Key, 0) + 1
+            if Seen[Key] <= (1 if Kind == "Line" else 2):
+                Definitions.append((Match.start(), Match.end()))
+                continue
+        Found.append((Match.start(), Match.end()))
+    return sorted(Found), Definitions
+
+
+def ResolveSpans(RelativePath: str, Content: str, Files: Access) -> list[dict]:
+    """[{"From", "To", "Text"}]: each variable's place in Content (character
+    offsets) and what it resolves to ("" for a definition's own marker).
+    Ones that stay as written (a portal to a missing file) are left out.
+
+    Each variable is wrapped in private-use marker characters, the note is
+    resolved as usual, and the markers say which result is whose. Results
+    copied into others (a definition used later) carry their markers along,
+    nested, and are skipped."""
+    Variables, Definitions = _TopLevelVariables(Content)
+    # Definitions aren't marked: markers beside them would stop the resolver
+    # trimming the lines around them.
+    Spans = [{"From": Start, "To": End, "Text": ""} for Start, End in Definitions]
+    if not Variables:
+        return Spans
+    Parts: list[str] = []
+    Cursor = 0
+    for Number, (Start, End) in enumerate(Variables):
+        Parts += [Content[Cursor:Start], f"\ue000{Number}\ue001", Content[Start:End], f"\ue002{Number}\ue003"]
+        Cursor = End
+    Parts.append(Content[Cursor:])
+    Resolved = ResolvePortals(RelativePath, "".join(Parts), Files)
+
+    Results: dict[int, str] = {}
+    Open: list[tuple[int, int]] = []  # (number, where its text starts)
+    for Token in _Token.finditer(Resolved):
+        if Token.group(1) is not None:
+            Open.append((int(Token.group(1)), Token.end()))
+            continue
+        Number = int(Token.group(2))
+        if all(Number != Waiting for Waiting, _ in Open):
+            continue  # a stray close, copied in with a definition's text
+        while Open[-1][0] != Number:
+            Open.pop()  # stray opens, likewise
+        _, TextStart = Open.pop()
+        if not Open and Number not in Results:
+            Results[Number] = _Token.sub("", Resolved[TextStart:Token.start()])
+
+    Spans += [
+        {"From": Start, "To": End, "Text": Results[Number]}
+        for Number, (Start, End) in enumerate(Variables)
+        if Number in Results and Results[Number] != Content[Start:End]
+    ]
+    return sorted(Spans, key=lambda Span: Span["From"])