Files
SaraView/Client/Graph.py
T

542 lines
25 KiB
Python

"""The graph view, like Obsidian's (and the web view's GraphPane.jsx): every
directory and file under a directory as a dot (directories in the heading
colours by depth, files White), joined to its parent directory and to every file that references it.
On a server the graph comes from /api/graph; locally BuildLocalGraph makes
the same one with the same reference rules as Backend/Graph.py. The layout is
the web view's force simulation (same constants), in numpy, drawn as it
settles. Scroll to zoom, hover a dot to light up its links; one mouse button
pans and the other opens a dot (Settings > Graph)."""
import math
import re
import threading
from typing import Callable, Optional
from urllib.parse import unquote
import flet as ft
import flet.canvas as cv
import numpy as np
import Core
from App import App
from Core import Palette
from Sources import JoinPath, NormalizeRelativePath, SourceError
MaxNodes = 2000
MaxReadBytes = 1_000_000
ReferenceFileTypes = {"markdown", "text"}
# Same patterns as Backend/Graph.py.
FencedCodePattern = re.compile(r"^ {0,3}(```|~~~).*?^ {0,3}\1", re.MULTILINE | re.DOTALL)
MarkdownLinkPattern = re.compile(r"!?\[[^\]\n]*\]\(\s*<?([^)\s>]+)>?(?:\s+\"[^\"]*\")?\s*\)")
WikiLinkPattern = re.compile(r"!?\[\[([^\]|#\n]+)(?:#[^\]|\n]*)?(?:\|[^\]\n]*)?\]\]")
ExternalPattern = re.compile(r"^([a-z][a-z0-9+.-]*:|//|#)", re.IGNORECASE)
# The path part of a portal: {Path ***Name}, {Path head 3}, {Path '12'}
# (Backend/VarFunctions.py's MarkerPattern, just the parts with a path).
PortalPathPattern = re.compile(r"\{([^\s{}'*]+)\s+(?:\*\*\*[^\s*{}]+|(?:head|tail)\s+\d+|'\d+')[^{}]*\}")
# Layout, same as GraphPane.jsx.
ChildLength = 50
ReferenceLength = 80
Repulsion = 900
CellSize = 160
TicksPerDraw = 12
# --- Building the graph (local files) -------------------------------------
def FindReferences(Content: str) -> list[tuple[str, str]]:
Content = FencedCodePattern.sub("", Content)
Found: list[tuple[str, str]] = []
for Match in MarkdownLinkPattern.finditer(Content):
Target = Match.group(1)
if ExternalPattern.match(Target):
continue
Target = unquote(Target.split("#", 1)[0].split("?", 1)[0])
if Target:
Found.append(("Path", Target))
for Match in WikiLinkPattern.finditer(Content):
Found.append(("Name", Match.group(1).strip()))
for Match in PortalPathPattern.finditer(Content):
Found.append(("Path", Match.group(1)))
return Found
def _Distance(Directory: str, Id: str) -> int:
Here = Directory.split("/") if Directory else []
There = Id.split("/")[:-1]
Shared = 0
for A, B in zip(Here, There):
if A != B:
break
Shared += 1
return len(Here) + len(There) - 2 * Shared
def ResolveReferences(References: list[tuple[str, str]], FileId: str, Ids: set[str],
ByName: dict[str, list[str]]) -> set[str]:
Directory = FileId.rsplit("/", 1)[0] if "/" in FileId else ""
Targets: set[str] = set()
for Kind, Target in References:
if Kind == "Path":
Resolved = NormalizeRelativePath("" if Target.startswith("/") else Directory, Target)
if Resolved in Ids:
Targets.add(Resolved)
continue
Found = next((NormalizeRelativePath(Base, Candidate)
for Candidate in (Target, f"{Target}.md") for Base in (Directory, "")
if NormalizeRelativePath(Base, Candidate) in Ids), None)
if Found is None:
Matches = ByName.get(Target.rsplit("/", 1)[-1].lower(), [])
if Matches:
Found = min(Matches, key=lambda Id: (_Distance(Directory, Id), Id))
if Found is not None:
Targets.add(Found)
Targets.discard(FileId)
return Targets
def BuildLocalGraph(Source, Root: str, Path: str) -> dict:
"""Backend/Graph.py's BuildGraph, through a Source's own listing (so the
same files are hidden as in the file list)."""
Nodes = [{"Id": Path, "Name": Path.rsplit("/", 1)[-1] if Path else Root, "IsDirectory": True, "FileType": None}]
Links: list[dict] = []
Truncated = False
Pending = [Path]
while Pending and not Truncated:
Directory = Pending.pop(0)
try:
Items = Source.List(Root, Directory)
except SourceError:
continue
for Item in Items:
if len(Nodes) >= MaxNodes:
Truncated = True
break
Id = JoinPath(Directory, Item.Name)
Nodes.append({"Id": Id, "Name": Item.Name, "IsDirectory": Item.IsDirectory, "FileType": Item.FileType})
Links.append({"Source": Directory, "Target": Id, "Kind": "Child"})
if Item.IsDirectory:
Pending.append(Id)
Ids = {Node["Id"] for Node in Nodes}
ByName: dict[str, list[str]] = {}
for Node in Nodes:
if not Node["IsDirectory"]:
Name = Node["Name"].lower()
ByName.setdefault(Name, []).append(Node["Id"])
if "." in Name:
ByName.setdefault(Name.rsplit(".", 1)[0], []).append(Node["Id"])
for Node in Nodes:
if Node["IsDirectory"] or Node["FileType"] not in ReferenceFileTypes:
continue
try:
if Source.LocalFile(Root, Node["Id"]).stat().st_size > MaxReadBytes:
continue
Content = Source.ReadText(Root, Node["Id"])
except (SourceError, OSError):
continue
for Target in sorted(ResolveReferences(FindReferences(Content), Node["Id"], Ids, ByName)):
Links.append({"Source": Node["Id"], "Target": Target, "Kind": "Reference"})
return {"Root": Root, "Path": Path, "Nodes": Nodes, "Links": Links, "Truncated": Truncated}
# --- Layout ----------------------------------------------------------------
class Layout:
def __init__(self, Graph: dict):
self.Nodes = Graph["Nodes"]
Index = {Node["Id"]: Number for Number, Node in enumerate(self.Nodes)}
Links = [Link for Link in Graph["Links"] if Link["Source"] in Index and Link["Target"] in Index]
self.Kinds = [Link["Kind"] for Link in Links]
self.Sources = np.array([Index[Link["Source"]] for Link in Links], dtype=int)
self.Targets = np.array([Index[Link["Target"]] for Link in Links], dtype=int)
IsChild = np.array([Kind == "Child" for Kind in self.Kinds], dtype=bool)
self.Rest = np.where(IsChild, ChildLength, ReferenceLength).astype(float)
self.Strength = np.where(IsChild, 0.08, 0.03)
Count = len(self.Nodes)
self.Degree = np.bincount(np.concatenate([self.Sources, self.Targets]), minlength=Count) if Links else np.zeros(Count)
self.Neighbours: list[set[int]] = [set() for _ in range(Count)]
for A, B in zip(self.Sources, self.Targets):
self.Neighbours[A].add(int(B))
self.Neighbours[B].add(int(A))
self.Position = self._Initial(Links, Index)
self.Velocity = np.zeros_like(self.Position)
self.Alpha = 1.0
def _Initial(self, Links: list[dict], Index: dict[str, int]) -> np.ndarray:
"""Each dot near its parent, spread around it (seeded, like the web view)."""
Random = np.random.default_rng(12345)
Position = np.zeros((len(self.Nodes), 2))
Placed = np.zeros(len(self.Nodes), dtype=bool)
Placed[0] = True
Children: dict[int, list[int]] = {}
for Link in Links:
if Link["Kind"] == "Child":
Children.setdefault(Index[Link["Source"]], []).append(Index[Link["Target"]])
Queue = [0]
while Queue:
Parent = Queue.pop(0)
Kids = Children.get(Parent, [])
Start = Random.random() * math.tau
for Number, Child in enumerate(Kids):
Angle = Start + Number / len(Kids) * math.tau
Distance = ChildLength * (1 + Random.random() * 0.5)
Position[Child] = Position[Parent] + (math.cos(Angle) * Distance, math.sin(Angle) * Distance)
Placed[Child] = True
Queue.append(Child)
Loose = ~Placed
Position[Loose] = (Random.random((Loose.sum(), 2)) - 0.5) * 200
return Position
@property
def Settled(self) -> bool:
return self.Alpha <= 0.005
def Tick(self) -> None:
Position, Velocity, Alpha = self.Position, self.Velocity, self.Alpha
Count = len(Position)
for Start in range(0, Count, 256): # in chunks, so memory stays small for big graphs
Delta = Position[Start:Start + 256, None, :] - Position[None, :, :]
DistanceSquared = np.maximum((Delta ** 2).sum(-1), 1.0)
Force = Repulsion * Alpha / DistanceSquared
Force[DistanceSquared > CellSize * CellSize] = 0
Velocity[Start:Start + 256] += (Delta * Force[..., None]).sum(1)
if len(self.Sources):
Delta = Position[self.Targets] - Position[self.Sources]
Distance = np.maximum(np.sqrt((Delta ** 2).sum(-1)), 1e-6)
Pull = ((Distance - self.Rest) / Distance * self.Strength * Alpha)[:, None] * Delta
np.add.at(Velocity, self.Sources, Pull)
np.add.at(Velocity, self.Targets, -Pull)
Velocity -= Position * 0.002 * Alpha
Velocity *= 0.6
Position += Velocity
self.Alpha *= 0.985
# --- View --------------------------------------------------------------------
# Directories go down the heading colours by depth (h1 Mauve, h2 Blue, ...),
# starting again after Red. Files are White.
DepthColours = ["Mauve", "Blue", "Green", "Yellow", "Peach", "Red"]
# With GraphShrinkByDepth, directory dots shrink in even steps from their
# usual size to a file dot's size (a file with one link) at this depth (the
# red ones), and stay that size below it.
ShrinkDepth = 5
FileDotRadius = 3.8
def DirectoryColour(Depth: int) -> str:
return getattr(Palette, DepthColours[Depth % len(DepthColours)])
def FilterGraph(Graph: dict, ShowDirectories: bool) -> dict:
"""Without directories: just the files and the references between them
(files with none float on their own)."""
if ShowDirectories:
return Graph
Nodes = [Node for Node in Graph["Nodes"] if not Node["IsDirectory"]]
Links = [Link for Link in Graph["Links"] if Link["Kind"] == "Reference"]
return {**Graph, "Nodes": Nodes, "Links": Links}
class GraphView(ft.Container):
"""Pan and zoom are done here rather than with an InteractiveViewer, so
the buttons can be picked (Settings > Graph): one mouse button pans, the
other opens the dot it clicks."""
def __init__(self, TheApp: App, Root: str, DirectoryPath: str,
OnOpenNode: Callable[[dict], None], OnClose: Callable[[], None]):
self.TheApp = TheApp
self.Root = Root
self.DirectoryPath = DirectoryPath
self.OnOpenNode = OnOpenNode
# Anything but "Left" (including an old "Middle") pans with Right.
self.PanButton = "Left" if TheApp.Config.GraphPanButton == "Left" else "Right"
self.OpenButton = "Right" if self.PanButton == "Left" else "Left"
self.ShowDirectories = TheApp.Config.GraphShowDirectories
self.ColourByDepth = TheApp.Config.GraphColourByDepth
self.ShrinkByDepth = TheApp.Config.GraphShrinkByDepth
self.Depths = np.zeros(0) # each dot's depth below the graph's directory
self.Layout: Optional[Layout] = None
self.ScopePath = ""
self.Size = (800.0, 600.0)
self.Screen = np.zeros((0, 2)) # each dot's position on the canvas
self.Radius = np.zeros(0)
self.Colours: list[str] = []
self.Hovered: Optional[int] = None
self.Closed = False
self.Generation = 0 # bumps when the scope changes, so an old layout thread stops
# The view: canvas point P shows at P * Zoom + Pan.
self.Zoom = 1.0
self.PanX = 0.0
self.PanY = 0.0
self.DragFrom: Optional[tuple[float, float]] = None
self.ScopePicker = ft.Dropdown(
value="Root",
options=[ft.DropdownOption(key="Root", text="Whole root"),
ft.DropdownOption(key="Directory", text=f"This directory ({DirectoryPath})" if DirectoryPath else "This directory",
disabled=not DirectoryPath)],
dense=True, width=260, text_size=13,
on_select=lambda e: self.Reload(),
)
self.CountText = ft.Text("", size=12, color=Palette.Overlay0)
LegendItems: list[ft.Control] = []
if self.ShowDirectories and not self.ColourByDepth:
LegendItems += [ft.Text("●", color=Palette.Mauve), ft.Text("Directory", size=12, color=Palette.Overlay0)]
elif self.ShowDirectories:
LegendItems += [ft.Text("●", color=Palette.Mauve), ft.Text("●", color=Palette.Blue), ft.Text("●", color=Palette.Green),
ft.Text("Directories by depth", size=12, color=Palette.Overlay0)]
LegendItems += [ft.Text("●", color=Palette.White), ft.Text("File", size=12, color=Palette.Overlay0),
ft.Text("─", color=Palette.Blue), ft.Text("Reference", size=12, color=Palette.Overlay0)]
self.BaseCanvas = cv.Canvas(shapes=[], expand=True, on_resize=self._OnResize, resize_interval=100)
self.LitCanvas = cv.Canvas(shapes=[], expand=True)
self.Plane = ft.Container(content=ft.Stack([self.BaseCanvas, self.LitCanvas], expand=True), expand=True)
self.Status = ft.Text("Building the graph…", color=Palette.Overlay0)
# Hidden (not just its text) once the graph is up: even empty, it sits
# over the graph and would catch every click.
self.StatusBox = ft.Container(self.Status, alignment=ft.Alignment.CENTER)
Hint = ("Drag to pan, right-click a dot to open it" if self.PanButton == "Left"
else "Right-drag to pan, click a dot to open it")
self.Surface = ft.GestureDetector(
content=ft.Container(self.Plane, expand=True, clip_behavior=ft.ClipBehavior.HARD_EDGE, bgcolor=Palette.Base),
on_tap_up=lambda e: self._OnClick(e, "Left"),
on_secondary_tap_up=lambda e: self._OnClick(e, "Right"),
on_hover=self._OnHover,
on_exit=lambda e: self._SetHovered(None),
on_scroll=self._OnScroll,
on_pan_down=lambda e: self._StartDrag(e, "Left"), # down, not start: start comes after the drag slop
on_pan_update=lambda e: self._Drag(e, "Left"),
on_pan_end=lambda e: self._EndDrag(),
on_right_pan_start=lambda e: self._StartDrag(e, "Right"),
on_right_pan_update=lambda e: self._Drag(e, "Right"),
on_right_pan_end=lambda e: self._EndDrag(),
hover_interval=20,
drag_interval=16,
expand=True,
)
super().__init__(
content=ft.Column([
ft.Row([
ft.IconButton(icon=ft.Icons.ARROW_BACK, tooltip="Back to the file list (Esc)", on_click=lambda e: OnClose()),
ft.Text("Graph", weight=ft.FontWeight.BOLD),
self.ScopePicker, ft.Row(LegendItems, spacing=4, tight=True),
ft.Container(expand=True), self.CountText,
], spacing=12),
ft.Divider(height=1, color=Palette.Surface1),
ft.Stack([self.Surface, self.StatusBox], expand=True),
ft.Text(f"{Hint}, scroll to zoom.", size=11, color=Palette.Overlay0),
], spacing=8, expand=True),
expand=True, bgcolor=Palette.Base, padding=12,
)
def did_mount(self) -> None:
self.Reload()
def will_unmount(self) -> None:
self.Closed = True
# --- Data and layout ----------------------------------------------------
def Reload(self) -> None:
self.Generation += 1
Generation = self.Generation
self.ScopePath = self.DirectoryPath if self.ScopePicker.value == "Directory" else ""
self.Status.value = "Building the graph…"
self.StatusBox.visible = True
self.BaseCanvas.shapes = []
self.LitCanvas.shapes = []
self.Layout = None
self.Hovered = None
self.update()
self.page.run_thread(self._Build, Generation, self.ScopePath)
def _Build(self, Generation: int, Path: str) -> None:
try:
Graph = self.TheApp.Source.Graph(self.Root, Path)
except SourceError as Error:
if Generation == self.Generation and not self.Closed:
self.Status.value = f"Couldn't build the graph: {Error}"
self.Status.update()
return
if Generation != self.Generation or self.Closed:
return
Directories = sum(1 for Node in Graph["Nodes"] if Node["IsDirectory"])
Files = len(Graph["Nodes"]) - Directories
Graph = FilterGraph(Graph, self.ShowDirectories)
Count = f"{Directories} directories, {Files} files" if self.ShowDirectories else f"{Files} files"
self.CountText.value = Count + (" (cut short: too many to show)" if Graph.get("Truncated") else "")
self.CountText.update()
if not Graph["Nodes"]:
self.Status.value = "Nothing to show here."
self.Status.update()
return
# This thread keeps its own Layout: a Reload meanwhile (the graph
# opening, or the scope changing) clears self.Layout for its own thread.
Settling = Layout(Graph)
Base = len(Path.split("/")) if Path else 0
Depths = np.array([(len(Node["Id"].split("/")) if Node["Id"] else 0) - Base for Node in Settling.Nodes])
Colours = [(DirectoryColour(int(Depth)) if self.ColourByDepth else Palette.Mauve) if Node["IsDirectory"]
else Palette.White for Node, Depth in zip(Settling.Nodes, Depths)]
if Generation != self.Generation or self.Closed:
return
self.Layout, self.Depths, self.Colours = Settling, Depths, Colours
self.StatusBox.visible = False
self.StatusBox.update()
# Settle in steps, drawing as it goes, like the web view's animation.
while not Settling.Settled:
for _ in range(TicksPerDraw):
Settling.Tick()
if Generation != self.Generation or self.Closed:
return
self._Draw()
self._Draw()
# --- Drawing ------------------------------------------------------------
def _Fit(self) -> None:
"""Maps the layout onto the canvas, so the whole graph fits at zoom 1."""
Position = self.Layout.Position
Low, High = Position.min(0), Position.max(0)
Width, Height = self.Size
Margin = 40
Scale = min((Width - 2 * Margin) / max(High[0] - Low[0], 1), (Height - 2 * Margin) / max(High[1] - Low[1], 1), 1.5)
Scale = max(Scale, 0.05) # the canvas can start out smaller than the margins
Middle = (Low + High) / 2
self.Screen = (Position - Middle) * Scale + (Width / 2, Height / 2)
Zoom = math.sqrt(Scale)
IsDirectory = np.array([Node["IsDirectory"] for Node in self.Layout.Nodes])
Degree = np.sqrt(self.Layout.Degree)
DirectoryRadius = 5 + Degree * 1.6
if self.ShrinkByDepth:
Step = np.minimum(self.Depths, ShrinkDepth) / ShrinkDepth
DirectoryRadius = DirectoryRadius + (FileDotRadius - DirectoryRadius) * Step
self.Radius = np.maximum(2, np.where(IsDirectory, DirectoryRadius, 3 + Degree * 0.8) * Zoom)
def _Label(self, Number: int, Colour: str) -> cv.Text:
X, Y = self.Screen[Number]
return cv.Text(X, Y + self.Radius[Number] + 2, self.Layout.Nodes[Number]["Name"],
style=ft.TextStyle(size=10, color=Colour, font_family=Core.ActiveFontFamily),
alignment=ft.Alignment.TOP_CENTER)
def _Dot(self, Number: int) -> cv.Circle:
X, Y = self.Screen[Number]
return cv.Circle(X, Y, self.Radius[Number], paint=ft.Paint(color=self.Colours[Number]))
def _Line(self, LinkNumber: int, Colour: str, Width: float = 1) -> cv.Line:
A, B = self.Screen[self.Layout.Sources[LinkNumber]], self.Screen[self.Layout.Targets[LinkNumber]]
return cv.Line(A[0], A[1], B[0], B[1], paint=ft.Paint(color=Colour, stroke_width=Width, style=ft.PaintingStyle.STROKE))
def _Draw(self) -> None:
if self.Layout is None or self.Closed:
return
self._Fit()
Shapes: list = []
for Number, Kind in enumerate(self.Layout.Kinds):
Shapes.append(self._Line(Number, Palette.Surface1 if Kind == "Child" else Palette.Blue))
Nodes = self.Layout.Nodes
ShowFileLabels = len(Nodes) <= 300
for Number, Node in enumerate(Nodes):
Shapes.append(self._Dot(Number))
if Node["IsDirectory"] or ShowFileLabels:
Shapes.append(self._Label(Number, Palette.Overlay0))
self.BaseCanvas.shapes = Shapes
self.BaseCanvas.update()
self._DrawHighlight()
def _DrawHighlight(self) -> None:
"""The hovered dot and its links, over a veil that dims the rest."""
Number = self.Hovered
if Number is None or self.Layout is None:
self.LitCanvas.shapes = []
else:
Width, Height = self.Size
Shapes: list = [cv.Rect(-4000, -4000, Width + 8000, Height + 8000,
paint=ft.Paint(color=ft.Colors.with_opacity(0.75, Palette.Base)))]
Lit = {Number} | self.Layout.Neighbours[Number]
for LinkNumber in range(len(self.Layout.Kinds)):
if Number in (self.Layout.Sources[LinkNumber], self.Layout.Targets[LinkNumber]):
Shapes.append(self._Line(LinkNumber, Palette.Mauve, 1.5))
for Other in Lit:
Shapes.append(self._Dot(Other))
Shapes.append(self._Label(Other, Palette.Text if Other == Number else Palette.Overlay0))
X, Y = self.Screen[Number]
Shapes.append(cv.Circle(X, Y, self.Radius[Number] + 3,
paint=ft.Paint(color=Palette.Mauve, stroke_width=2, style=ft.PaintingStyle.STROKE)))
self.LitCanvas.shapes = Shapes
self.LitCanvas.update()
# --- View transform -----------------------------------------------------
def _ApplyView(self) -> None:
self.Plane.transform = ft.Transform(
matrix=ft.Matrix4.identity().translate(self.PanX, self.PanY).scale(self.Zoom, self.Zoom),
alignment=ft.Alignment.TOP_LEFT,
)
self.Plane.update()
def _ToCanvas(self, X: float, Y: float) -> tuple[float, float]:
return (X - self.PanX) / self.Zoom, (Y - self.PanY) / self.Zoom
def _PanBy(self, DX: float, DY: float) -> None:
self.PanX += DX
self.PanY += DY
self._ApplyView()
# --- Input --------------------------------------------------------------
def _NodeAt(self, X: float, Y: float) -> Optional[int]:
"""The dot under a point on the surface (screen coordinates)."""
if self.Layout is None or not len(self.Screen):
return None
X, Y = self._ToCanvas(X, Y)
Distance = np.hypot(self.Screen[:, 0] - X, self.Screen[:, 1] - Y)
Number = int(Distance.argmin())
Reach = max(6 / self.Zoom, self.Radius[Number] + 4 / self.Zoom) # at least 6 screen pixels
return Number if Distance[Number] <= Reach else None
def _OnResize(self, Event) -> None:
self.Size = (Event.width, Event.height)
self._Draw()
def _SetHovered(self, Number: Optional[int]) -> None:
if Number != self.Hovered:
self.Hovered = Number
self._DrawHighlight()
def _OnHover(self, Event) -> None:
X, Y = Event.local_position.x, Event.local_position.y
self._SetHovered(self._NodeAt(X, Y))
def _OnClick(self, Event, Button: str) -> None:
if Button != self.OpenButton:
return
Number = self._NodeAt(Event.local_position.x, Event.local_position.y)
if Number is not None:
self.OnOpenNode(self.Layout.Nodes[Number])
def _StartDrag(self, Event, Button: str) -> None:
if Button == self.PanButton:
self.DragFrom = (Event.local_position.x, Event.local_position.y)
def _Drag(self, Event, Button: str) -> None:
if Button != self.PanButton or self.DragFrom is None:
return
X, Y = Event.local_position.x, Event.local_position.y
self._PanBy(X - self.DragFrom[0], Y - self.DragFrom[1])
self.DragFrom = (X, Y)
def _EndDrag(self) -> None:
self.DragFrom = None
def _OnScroll(self, Event) -> None:
"""Zooms around the pointer."""
X, Y = Event.local_position.x, Event.local_position.y
CanvasX, CanvasY = self._ToCanvas(X, Y)
self.Zoom = min(8.0, max(0.2, self.Zoom * math.exp(-Event.scroll_delta.y * 0.0015)))
self.PanX = X - CanvasX * self.Zoom
self.PanY = Y - CanvasY * self.Zoom
self._ApplyView()