Client: variables resolved in the live editor and reading view

Co-Authored-By: Claude Opus 5.5 <[email protected]>

Client/LiveMarkdown.py

@@ -9,7 +9,9 @@ list, table or quote). The text between blocks (blank lines) is kept
 byte for byte, so editing one block never reformats the rest of the file.
 A ```mermaid block shows as its diagram when MakeDiagram gives one (Diagrams.py),
 and the properties block at the top as its panel (Properties.py). Checkboxes
-(tasks and true/false properties) tick without opening their block."""
+(tasks and true/false properties) tick without opening their block. With
+GetSpans, text and headings show their variables' results (Variables.py);
+opening a block shows what's written."""
 
 import re
 from dataclasses import dataclass
@@ -20,6 +22,7 @@ import flet as ft
 import Core
 from Core import CodeStyleSheet, MarkdownCodeTheme, MarkdownSheet, Palette
 from Properties import ParseProperties, PropertiesView, ToggleLine, WithCheckboxes
+from Variables import Substitute
 
 FenceStart = re.compile(r"^ {0,3}(`{3,}|~{3,})")
 HeadingLine = re.compile(r"^ {0,3}(#{1,6})(\s|$)")
@@ -113,7 +116,8 @@ def SplitBlocks(Document: str) -> list[Block]:
 class LiveMarkdown(ft.Column):
     def __init__(self, Document: str, FontSize: int, OnChange: Callable[[], None],
                  OnFocusChange: Callable[[bool], None], OnTapLink: Callable,
-                 MakeDiagram: Optional[Callable[[str, int], Optional[ft.Control]]] = None):
+                 MakeDiagram: Optional[Callable[[str, int], Optional[ft.Control]]] = None,
+                 GetSpans: Optional[Callable[[str], list[dict]]] = None):
         super().__init__(scroll=ft.ScrollMode.AUTO, expand=True, spacing=0,
                          horizontal_alignment=ft.CrossAxisAlignment.STRETCH)
         self.Document = Document
@@ -122,11 +126,13 @@ class LiveMarkdown(ft.Column):
         self.OnFocusChange = OnFocusChange
         self.OnTapLink = OnTapLink
         self.MakeDiagram = MakeDiagram
+        self.GetSpans = GetSpans
+        self.SpansFor: tuple[str, list[dict]] = ("", [])  # (document, its spans)
         self.Blocks: list[Block] = []
         self.BlockControls: list[ft.Control] = []
         # Rendered blocks by their text, reused when the layout is rebuilt so
         # only the edited part is rendered again.
-        self.RenderCache: dict[tuple[str, str], list[ft.Control]] = {}
+        self.RenderCache: dict[tuple[str, str], list[ft.Control]] = {}  # by (kind, text as shown)
         self.EditingIndex: Optional[int] = None  # len(Blocks) is the new block at the end
         self.Field: Optional[ft.TextField] = None
         self._Layout(Update=False)
@@ -239,7 +245,14 @@ class LiveMarkdown(ft.Column):
             on_tap_link=self.OnTapLink,
         )
 
-    def _Render(self, Text: str, Kind: str) -> ft.Control:
+    def _Spans(self) -> list[dict]:
+        if self.GetSpans is None:
+            return []
+        if self.SpansFor[0] != self.Document:
+            self.SpansFor = (self.Document, self.GetSpans(self.Document))
+        return self.SpansFor[1]
+
+    def _Render(self, Text: str, Kind: str, Shown: Optional[str] = None, LineMap: Optional[list] = None) -> ft.Control:
         Rendered = ft.Container(padding=ft.Padding.symmetric(horizontal=8, vertical=6), border_radius=4)
         # Lines are counted from the block's start, found again when ticked:
         # a rendered block is reused wherever its text ends up (RenderCache).
@@ -250,10 +263,10 @@ class LiveMarkdown(ft.Column):
         elif Kind == "Properties" and (Found := ParseProperties(Text)) is not None:
             Rendered.content = PropertiesView(Found, self.FontSize, self._Markdown, Toggle)
         elif Kind == "Text":
-            Parts = WithCheckboxes(Text, 0, self._Markdown, Toggle, self.FontSize)
+            Parts = WithCheckboxes(Shown if Shown is not None else Text, 0, self._Markdown, Toggle, self.FontSize, LineMap)
             Rendered.content = Parts[0] if len(Parts) == 1 else ft.Column(Parts, spacing=0, tight=True)
         else:
-            Rendered.content = self._Markdown(Text)
+            Rendered.content = self._Markdown(Shown if Shown is not None and Kind == "Heading" else Text)
         Rendered.on_click = lambda e, C=Rendered: self._OnBlockClicked(C)
         Rendered.on_hover = lambda e, C=Rendered: self._OnHover(C, e)
         return Rendered
@@ -276,6 +289,7 @@ class LiveMarkdown(ft.Column):
 
     def _Layout(self, Update: bool = True) -> None:
         self.Blocks = SplitBlocks(self.Document)
+        Spans = self._Spans()
         Cache = self.RenderCache
         self.RenderCache = {}
         self.BlockControls = []
@@ -284,11 +298,12 @@ class LiveMarkdown(ft.Column):
                 self.BlockControls.append(self.Field)
                 continue
             Text = self.Document[Item.Start:Item.End]
-            Reused = Cache.get((Item.Kind, Text))
+            Shown, LineMap = Substitute(Text, Item.Start, Spans) if Item.Kind in ("Text", "Heading") else (Text, None)
-            Control = Reused.pop() if Reused else self._Render(Text, Item.Kind)
+            Reused = Cache.get((Item.Kind, Shown))
+            Control = Reused.pop() if Reused else self._Render(Text, Item.Kind, Shown, LineMap)
             Control.bgcolor = None
             self.BlockControls.append(Control)
-            self.RenderCache.setdefault((Item.Kind, Text), []).append(Control)
+            self.RenderCache.setdefault((Item.Kind, Shown), []).append(Control)
         Controls = list(self.BlockControls)
         if self.EditingIndex == len(self.Blocks):
             Controls.append(self.Field)

Client/Packaging/BuildAppImage.sh

@@ -12,6 +12,7 @@ cd "$(dirname "$(readlink -f "$0")")/.."
 .venv/bin/pyinstaller --noconfirm --clean --log-level WARN \
     --onefile --windowed --name SaraView --collect-data flet \
     --add-data "$PWD/assets:assets" \
+    --paths "$PWD/../Backend" --hidden-import VarEngine \
     --distpath build/PyInstaller/dist --workpath build/PyInstaller/work --specpath build/PyInstaller \
     "$PWD/Main.py"
 

Client/Packaging/SaraSuite/Manifest.toml

@@ -6,7 +6,8 @@ Description = "SaraView's desktop client: view and edit files in a local folder,
 [Source]
 Git = "{GitHost}/SaraView"
 # Only the Flet client; the server (Backend/, Frontend/) runs in Docker on SpineS.
-Include = ["Client/*.py", "Client/requirements.txt", "Client/assets/**"]
+# Backend/VarEngine.py is the variable engine the client shares with it.
+Include = ["Client/*.py", "Client/requirements.txt", "Client/assets/**", "Backend/VarEngine.py"]
 Exclude = ["Client/Packaging"]
 
 [Python]

Client/Properties.py

@@ -169,9 +169,11 @@ def PropertiesView(Found: Parsed, FontSize: int, MakeMarkdown: Callable[[str], f
 
 
 def WithCheckboxes(Text: str, FirstLine: int, MakeMarkdown: Callable[[str], ft.Control],
-                   Toggle: OnToggle, FontSize: int) -> list[ft.Control]:
+                   Toggle: OnToggle, FontSize: int, LineMap: Optional[list[Optional[int]]] = None) -> list[ft.Control]:
     """Markdown with its task lines as real checkboxes: the lines between
-    them render as usual. FirstLine is where Text starts in the note."""
+    them render as usual. FirstLine is where Text starts in the note; LineMap
+    (Variables.Substitute) says which of its lines each shown line is, None
+    for one from a variable's result (its checkbox can't be ticked)."""
     Controls: list[ft.Control] = []
     Pending: list[str] = []
     InFence = False
@@ -190,8 +192,9 @@ def WithCheckboxes(Text: str, FirstLine: int, MakeMarkdown: Callable[[str], ft.C
             continue
         Flush()
         Indent = len(Task.group(1)) - len(Task.group(1).lstrip())
+        SourceLine = Offset if LineMap is None else LineMap[Offset]
         Controls.append(ft.Row(
-            [_Checkbox(Task.group(2) != " ", FirstLine + Offset, Toggle),
+            [_Checkbox(Task.group(2) != " ", FirstLine + (SourceLine or 0), Toggle if SourceLine is not None else None),
              ft.Container(MakeMarkdown(Line[Task.end():].strip() or " "), expand=True)],
             spacing=4,
             vertical_alignment=ft.CrossAxisAlignment.CENTER,

Client/Sources.py

@@ -543,6 +543,10 @@ class ServerSource(Source):
     def RemoveLock(self, Root: str, RelativePath: str) -> None:
         self._Json("DELETE", "/lock", params=self._At(Root, RelativePath))
 
+    def Spans(self, Root: str, RelativePath: str, Content: str) -> list[dict]:
+        """Where Content's variables are and what they resolve to (Variables.py)."""
+        return self._Json("POST", "/file/spans", params=self._At(Root, RelativePath), json={"Content": Content})["Spans"]
+
     def Diagram(self, Source: str, ThemePalette: dict[str, str], FontSize: int) -> bytes:
         """A ```mermaid block as a PNG, drawn by the server (Diagrams.py)."""
         Body = {"Source": Source, "Palette": ThemePalette, "FontSize": FontSize}

Client/Variables.py

@@ -0,0 +1,124 @@
+"""SaraView's variables (portals, definitions, loops) in the client: shown as
+what they resolve to, like the web view. A server says where each one is and
+what it becomes (POST /api/file/spans); in a local folder the client works it
+out itself with the server's own engine, Backend/VarEngine.py, which is
+installed beside the client so both read a note the same way.
+
+Spans are [{"From", "To", "Text"}] in characters of the whole note; Text ""
+is a definition's own marker, which just disappears."""
+
+import sys
+from pathlib import Path
+from typing import Optional
+
+from Sources import SourceError
+
+# VarEngine.py lives in Backend/, next to Client/ (in the repository, and in
+# what SaraSuite installs).
+_BackendDir = Path(__file__).resolve().parent.parent / "Backend"
+if _BackendDir.is_dir() and str(_BackendDir) not in sys.path:
+    sys.path.append(str(_BackendDir))
+try:
+    import VarEngine
+except ImportError:
+    VarEngine = None  # an install without it: variables stay as written
+
+Span = dict
+LineMap = list[Optional[int]]
+
+
+class _LocalAccess:
+    """VarEngine's file access for a local folder (LocalSource): everything
+    in it can be read, as there are no locks."""
+
+    def __init__(self, Source, Root: str):
+        self.Source = Source
+        self.Root = Root
+
+    def ReadText(self, RelativePath: str) -> Optional[str]:
+        try:
+            FilePath = self.Source.LocalFile(self.Root, RelativePath)
+            if not FilePath.is_file() or FilePath.suffix.lower() not in (".md", ".markdown", ".txt"):
+                return None
+            return FilePath.read_text(encoding="utf-8")
+        except (SourceError, OSError, UnicodeDecodeError, ValueError):
+            return None
+
+    def List(self, DirectoryPath: str) -> Optional[list[dict]]:
+        try:
+            Entries = self.Source.List(self.Root, DirectoryPath)
+        except SourceError:
+            return None
+        # Like the server's engine: hidden files and .sv_ notes never count,
+        # whatever the file list is set to show.
+        return [{"Name": Item.Name, "IsDirectory": Item.IsDirectory, "FileType": Item.FileType}
+                for Item in Entries if not Item.Name.startswith(".")]
+
+    def IsDirectory(self, RelativePath: str) -> bool:
+        try:
+            return self.Source.LocalFile(self.Root, RelativePath).is_dir()
+        except (SourceError, ValueError):
+            return False
+
+
+def FindSpans(TheApp, Root: str, RelativePath: str, Content: str) -> list[Span]:
+    """Where the variables in Content are and what they resolve to ([] if
+    that can't be worked out: then they show as written)."""
+    if "{" not in Content:
+        return []
+    Source = TheApp.Source
+    try:
+        if Source.Kind == "Server":
+            return Source.Spans(Root, RelativePath, Content)
+        if VarEngine is None:
+            return []
+        return VarEngine.ResolveSpans(RelativePath, Content, _LocalAccess(Source, Root))
+    except (SourceError, OSError, ValueError, RecursionError):
+        return []
+
+
+def Substitute(Text: str, Start: int, Spans: list[Span]) -> tuple[str, LineMap]:
+    """Text (which starts at Start in the note) with its variables shown as
+    their results, and for each line of that, which line of Text it is
+    (None for a line with a result in it: a checkbox there isn't the note's).
+    A result shows where its variable starts; a variable running on past
+    Text's end (a loop over several blocks) is cut there, and the part of one
+    that started earlier just disappears."""
+    Pieces: list[tuple[str, bool]] = []  # (text, is a result)
+    SkippedLines: list[int] = []  # source newlines a result replaced, per result piece
+    Cursor = 0
+    for Item in Spans:
+        From, To = Item["From"] - Start, Item["To"] - Start
+        if To <= 0 or From >= len(Text) or From < Cursor and From >= 0:
+            continue
+        From, To = max(From, 0), min(To, len(Text))
+        Pieces.append((Text[Cursor:From], False))
+        Pieces.append((Item["Text"] if Item["From"] >= Start else "", True))
+        SkippedLines.append(Text.count("\n", From, To))
+        Cursor = To
+    Pieces.append((Text[Cursor:], False))
+
+    Shown: list[str] = []
+    Map: LineMap = []
+    SourceLine = 0
+    LineStart = 0  # the source line the current shown line began on
+    Tainted = False
+    Skips = iter(SkippedLines)
+    for Piece, IsResult in Pieces:
+        Shown.append(Piece)
+        if IsResult:
+            Tainted = Tainted or Piece != ""
+            for _ in range(Piece.count("\n")):
+                Map.append(None)
+            Skipped = next(Skips)
+            SourceLine += Skipped
+            Tainted = Tainted or Skipped > 0  # this line now runs on from a later one
+            continue
+        for Character in Piece:
+            if Character == "\n":
+                Map.append(None if Tainted else LineStart)
+                SourceLine += 1
+                LineStart = SourceLine
+                Tainted = False
+    Map.append(None if Tainted else LineStart)
+    return "".join(Shown), Map

Client/Viewer.py

@@ -3,7 +3,8 @@ EditorView/ImageView idea, with SaraView's View/Edit/Save toolbar).
 
 - Markdown is rendered in View; the other text types are shown as
   highlighted code. On a server, ```mermaid blocks show as diagrams (Diagrams.py),
-  and the properties block at the top as its panel (Properties.py).
+  the properties block at the top as its panel (Properties.py), and
+  variables (portals, loops) as what they resolve to (Variables.py).
   Checkboxes (tasks, true/false properties) tick from View and save at once,
   like the web view; in Live they leave the note unsaved, like typing. Edit is a plain monospace field with an explicit Save
   (Ctrl+S). On a server, saving records line blame for you, like the web view.
@@ -25,6 +26,7 @@ from App import App
 from Core import ChipButton, CodeStyleSheet, HumanSize, MarkdownCodeTheme, MarkdownSheet, Palette
 from Diagrams import MaybeDiagram, WithDiagrams
 from Properties import ParseProperties, PropertiesView, ToggleLine, WithCheckboxes
+from Variables import FindSpans, Substitute
 from LiveMarkdown import LiveMarkdown
 from Sources import Entry, SourceError, TextFileTypes
 
@@ -127,7 +129,8 @@ class FileView(ft.Container):
                 if self.IsLive:
                     self.Live = LiveMarkdown(self.Content, self.TheApp.Config.FontSize, self._OnTextChanged,
                                              self.OnFocusChange, self._OnTapLink,
-                                             MakeDiagram=lambda Text, Size: MaybeDiagram(self.TheApp, Text, Size))
+                                             MakeDiagram=lambda Text, Size: MaybeDiagram(self.TheApp, Text, Size),
+                                             GetSpans=lambda Document: FindSpans(self.TheApp, self.Root, self.RelativePath, Document))
                     return self.Live
                 return self._RenderedView()
             if self.FileType == "image":
@@ -145,6 +148,7 @@ class FileView(ft.Container):
             Toggle = self._ToggleAndSave if self.TheApp.Source.CanWrite else None
             Found = ParseProperties(Text)
             BodyStart = Found.EndOffset if Found else 0
+            Spans = FindSpans(self.TheApp, self.Root, self.RelativePath, Text)
 
             def MakeMarkdown(Part: str) -> ft.Control:
                 return ft.Markdown(
@@ -159,7 +163,8 @@ class FileView(ft.Container):
 
             def MakeWithCheckboxes(Part: str, Start: int) -> ft.Control:
                 FirstLine = Text[:BodyStart + Start].count("\n")
-                Parts = WithCheckboxes(Part, FirstLine, MakeMarkdown, Toggle, FontSize)
+                Shown, LineMap = Substitute(Part, BodyStart + Start, Spans)  # variables as their results
+                Parts = WithCheckboxes(Shown, FirstLine, MakeMarkdown, Toggle, FontSize, LineMap)
                 return Parts[0] if len(Parts) == 1 else ft.Column(Parts, spacing=0, tight=True)
 
             Views = WithDiagrams(self.TheApp, Text[BodyStart:], FontSize, MakeWithCheckboxes)