"""The graph view, like Obsidian's (and the web view's GraphPane.jsx): every directory and file under a directory as a dot (directories in the heading colours by depth, files White), joined to its parent directory and to every file that references it. On a server the graph comes from /api/graph; locally BuildLocalGraph makes the same one with the same reference rules as Backend/Graph.py. The layout is the web view's force simulation (same constants), in numpy, drawn as it settles. Scroll to zoom, hover a dot to light up its links; one mouse button pans and the other opens a dot (Settings > Graph).""" import math import re import threading from typing import Callable, Optional from urllib.parse import unquote import flet as ft import flet.canvas as cv import numpy as np import Core from App import App from Core import Palette from Sources import JoinPath, NormalizeRelativePath, SourceError MaxNodes = 2000 MaxReadBytes = 1_000_000 ReferenceFileTypes = {"markdown", "text"} # Same patterns as Backend/Graph.py. FencedCodePattern = re.compile(r"^ {0,3}(```|~~~).*?^ {0,3}\1", re.MULTILINE | re.DOTALL) MarkdownLinkPattern = re.compile(r"!?\[[^\]\n]*\]\(\s*]+)>?(?:\s+\"[^\"]*\")?\s*\)") WikiLinkPattern = re.compile(r"!?\[\[([^\]|#\n]+)(?:#[^\]|\n]*)?(?:\|[^\]\n]*)?\]\]") ExternalPattern = re.compile(r"^([a-z][a-z0-9+.-]*:|//|#)", re.IGNORECASE) # The path part of a portal: {Path ***Name}, {Path head 3}, {Path '12'} # (Backend/VarFunctions.py's MarkerPattern, just the parts with a path). PortalPathPattern = re.compile(r"\{([^\s{}'*]+)\s+(?:\*\*\*[^\s*{}]+|(?:head|tail)\s+\d+|'\d+')[^{}]*\}") # Layout, same as GraphPane.jsx. ChildLength = 50 ReferenceLength = 80 Repulsion = 900 CellSize = 160 TicksPerDraw = 12 # --- Building the graph (local files) ------------------------------------- def FindReferences(Content: str) -> list[tuple[str, str]]: Content = FencedCodePattern.sub("", Content) Found: list[tuple[str, str]] = [] for Match in MarkdownLinkPattern.finditer(Content): Target = Match.group(1) if ExternalPattern.match(Target): continue Target = unquote(Target.split("#", 1)[0].split("?", 1)[0]) if Target: Found.append(("Path", Target)) for Match in WikiLinkPattern.finditer(Content): Found.append(("Name", Match.group(1).strip())) for Match in PortalPathPattern.finditer(Content): Found.append(("Path", Match.group(1))) return Found def _Distance(Directory: str, Id: str) -> int: Here = Directory.split("/") if Directory else [] There = Id.split("/")[:-1] Shared = 0 for A, B in zip(Here, There): if A != B: break Shared += 1 return len(Here) + len(There) - 2 * Shared def ResolveReferences(References: list[tuple[str, str]], FileId: str, Ids: set[str], ByName: dict[str, list[str]]) -> set[str]: Directory = FileId.rsplit("/", 1)[0] if "/" in FileId else "" Targets: set[str] = set() for Kind, Target in References: if Kind == "Path": Resolved = NormalizeRelativePath("" if Target.startswith("/") else Directory, Target) if Resolved in Ids: Targets.add(Resolved) continue Found = next((NormalizeRelativePath(Base, Candidate) for Candidate in (Target, f"{Target}.md") for Base in (Directory, "") if NormalizeRelativePath(Base, Candidate) in Ids), None) if Found is None: Matches = ByName.get(Target.rsplit("/", 1)[-1].lower(), []) if Matches: Found = min(Matches, key=lambda Id: (_Distance(Directory, Id), Id)) if Found is not None: Targets.add(Found) Targets.discard(FileId) return Targets def BuildLocalGraph(Source, Root: str, Path: str) -> dict: """Backend/Graph.py's BuildGraph, through a Source's own listing (so the same files are hidden as in the file list).""" Nodes = [{"Id": Path, "Name": Path.rsplit("/", 1)[-1] if Path else Root, "IsDirectory": True, "FileType": None}] Links: list[dict] = [] Truncated = False Pending = [Path] while Pending and not Truncated: Directory = Pending.pop(0) try: Items = Source.List(Root, Directory) except SourceError: continue for Item in Items: if len(Nodes) >= MaxNodes: Truncated = True break Id = JoinPath(Directory, Item.Name) Nodes.append({"Id": Id, "Name": Item.Name, "IsDirectory": Item.IsDirectory, "FileType": Item.FileType}) Links.append({"Source": Directory, "Target": Id, "Kind": "Child"}) if Item.IsDirectory: Pending.append(Id) Ids = {Node["Id"] for Node in Nodes} ByName: dict[str, list[str]] = {} for Node in Nodes: if not Node["IsDirectory"]: Name = Node["Name"].lower() ByName.setdefault(Name, []).append(Node["Id"]) if "." in Name: ByName.setdefault(Name.rsplit(".", 1)[0], []).append(Node["Id"]) for Node in Nodes: if Node["IsDirectory"] or Node["FileType"] not in ReferenceFileTypes: continue try: if Source.LocalFile(Root, Node["Id"]).stat().st_size > MaxReadBytes: continue Content = Source.ReadText(Root, Node["Id"]) except (SourceError, OSError): continue for Target in sorted(ResolveReferences(FindReferences(Content), Node["Id"], Ids, ByName)): Links.append({"Source": Node["Id"], "Target": Target, "Kind": "Reference"}) return {"Root": Root, "Path": Path, "Nodes": Nodes, "Links": Links, "Truncated": Truncated} # --- Layout ---------------------------------------------------------------- class Layout: def __init__(self, Graph: dict): self.Nodes = Graph["Nodes"] Index = {Node["Id"]: Number for Number, Node in enumerate(self.Nodes)} Links = [Link for Link in Graph["Links"] if Link["Source"] in Index and Link["Target"] in Index] self.Kinds = [Link["Kind"] for Link in Links] self.Sources = np.array([Index[Link["Source"]] for Link in Links], dtype=int) self.Targets = np.array([Index[Link["Target"]] for Link in Links], dtype=int) IsChild = np.array([Kind == "Child" for Kind in self.Kinds], dtype=bool) self.Rest = np.where(IsChild, ChildLength, ReferenceLength).astype(float) self.Strength = np.where(IsChild, 0.08, 0.03) Count = len(self.Nodes) self.Degree = np.bincount(np.concatenate([self.Sources, self.Targets]), minlength=Count) if Links else np.zeros(Count) self.Neighbours: list[set[int]] = [set() for _ in range(Count)] for A, B in zip(self.Sources, self.Targets): self.Neighbours[A].add(int(B)) self.Neighbours[B].add(int(A)) self.Position = self._Initial(Links, Index) self.Velocity = np.zeros_like(self.Position) self.Alpha = 1.0 def _Initial(self, Links: list[dict], Index: dict[str, int]) -> np.ndarray: """Each dot near its parent, spread around it (seeded, like the web view).""" Random = np.random.default_rng(12345) Position = np.zeros((len(self.Nodes), 2)) Placed = np.zeros(len(self.Nodes), dtype=bool) Placed[0] = True Children: dict[int, list[int]] = {} for Link in Links: if Link["Kind"] == "Child": Children.setdefault(Index[Link["Source"]], []).append(Index[Link["Target"]]) Queue = [0] while Queue: Parent = Queue.pop(0) Kids = Children.get(Parent, []) Start = Random.random() * math.tau for Number, Child in enumerate(Kids): Angle = Start + Number / len(Kids) * math.tau Distance = ChildLength * (1 + Random.random() * 0.5) Position[Child] = Position[Parent] + (math.cos(Angle) * Distance, math.sin(Angle) * Distance) Placed[Child] = True Queue.append(Child) Loose = ~Placed Position[Loose] = (Random.random((Loose.sum(), 2)) - 0.5) * 200 return Position @property def Settled(self) -> bool: return self.Alpha <= 0.005 def Tick(self) -> None: Position, Velocity, Alpha = self.Position, self.Velocity, self.Alpha Count = len(Position) for Start in range(0, Count, 256): # in chunks, so memory stays small for big graphs Delta = Position[Start:Start + 256, None, :] - Position[None, :, :] DistanceSquared = np.maximum((Delta ** 2).sum(-1), 1.0) Force = Repulsion * Alpha / DistanceSquared Force[DistanceSquared > CellSize * CellSize] = 0 Velocity[Start:Start + 256] += (Delta * Force[..., None]).sum(1) if len(self.Sources): Delta = Position[self.Targets] - Position[self.Sources] Distance = np.maximum(np.sqrt((Delta ** 2).sum(-1)), 1e-6) Pull = ((Distance - self.Rest) / Distance * self.Strength * Alpha)[:, None] * Delta np.add.at(Velocity, self.Sources, Pull) np.add.at(Velocity, self.Targets, -Pull) Velocity -= Position * 0.002 * Alpha Velocity *= 0.6 Position += Velocity self.Alpha *= 0.985 # --- View -------------------------------------------------------------------- # Directories go down the heading colours by depth (h1 Mauve, h2 Blue, ...), # starting again after Red. Files are White. DepthColours = ["Mauve", "Blue", "Green", "Yellow", "Peach", "Red"] def DirectoryColour(Depth: int) -> str: return getattr(Palette, DepthColours[Depth % len(DepthColours)]) def FilterGraph(Graph: dict, ShowDirectories: bool) -> dict: """Without directories: just the files and the references between them (files with none float on their own).""" if ShowDirectories: return Graph Nodes = [Node for Node in Graph["Nodes"] if not Node["IsDirectory"]] Links = [Link for Link in Graph["Links"] if Link["Kind"] == "Reference"] return {**Graph, "Nodes": Nodes, "Links": Links} class GraphView(ft.Container): """Pan and zoom are done here rather than with an InteractiveViewer, so the buttons can be picked (Settings > Graph): one mouse button pans, the other opens the dot it clicks.""" def __init__(self, TheApp: App, Root: str, DirectoryPath: str, OnOpenNode: Callable[[dict], None], OnClose: Callable[[], None]): self.TheApp = TheApp self.Root = Root self.DirectoryPath = DirectoryPath self.OnOpenNode = OnOpenNode # Anything but "Left" (including an old "Middle") pans with Right. self.PanButton = "Left" if TheApp.Config.GraphPanButton == "Left" else "Right" self.OpenButton = "Right" if self.PanButton == "Left" else "Left" self.ShowDirectories = TheApp.Config.GraphShowDirectories self.Layout: Optional[Layout] = None self.ScopePath = "" self.Size = (800.0, 600.0) self.Screen = np.zeros((0, 2)) # each dot's position on the canvas self.Radius = np.zeros(0) self.Colours: list[str] = [] self.Hovered: Optional[int] = None self.Closed = False self.Generation = 0 # bumps when the scope changes, so an old layout thread stops # The view: canvas point P shows at P * Zoom + Pan. self.Zoom = 1.0 self.PanX = 0.0 self.PanY = 0.0 self.DragFrom: Optional[tuple[float, float]] = None self.ScopePicker = ft.Dropdown( value="Root", options=[ft.DropdownOption(key="Root", text="Whole root"), ft.DropdownOption(key="Directory", text=f"This directory ({DirectoryPath})" if DirectoryPath else "This directory", disabled=not DirectoryPath)], dense=True, width=260, text_size=13, on_select=lambda e: self.Reload(), ) self.CountText = ft.Text("", size=12, color=Palette.Overlay0) LegendItems: list[ft.Control] = [] if self.ShowDirectories: LegendItems += [ft.Text("●", color=Palette.Mauve), ft.Text("●", color=Palette.Blue), ft.Text("●", color=Palette.Green), ft.Text("Directories by depth", size=12, color=Palette.Overlay0)] LegendItems += [ft.Text("●", color=Palette.White), ft.Text("File", size=12, color=Palette.Overlay0), ft.Text("─", color=Palette.Blue), ft.Text("Reference", size=12, color=Palette.Overlay0)] self.BaseCanvas = cv.Canvas(shapes=[], expand=True, on_resize=self._OnResize, resize_interval=100) self.LitCanvas = cv.Canvas(shapes=[], expand=True) self.Plane = ft.Container(content=ft.Stack([self.BaseCanvas, self.LitCanvas], expand=True), expand=True) self.Status = ft.Text("Building the graph…", color=Palette.Overlay0) # Hidden (not just its text) once the graph is up: even empty, it sits # over the graph and would catch every click. self.StatusBox = ft.Container(self.Status, alignment=ft.Alignment.CENTER) Hint = ("Drag to pan, right-click a dot to open it" if self.PanButton == "Left" else "Right-drag to pan, click a dot to open it") self.Surface = ft.GestureDetector( content=ft.Container(self.Plane, expand=True, clip_behavior=ft.ClipBehavior.HARD_EDGE, bgcolor=Palette.Base), on_tap_up=lambda e: self._OnClick(e, "Left"), on_secondary_tap_up=lambda e: self._OnClick(e, "Right"), on_hover=self._OnHover, on_exit=lambda e: self._SetHovered(None), on_scroll=self._OnScroll, on_pan_down=lambda e: self._StartDrag(e, "Left"), # down, not start: start comes after the drag slop on_pan_update=lambda e: self._Drag(e, "Left"), on_pan_end=lambda e: self._EndDrag(), on_right_pan_start=lambda e: self._StartDrag(e, "Right"), on_right_pan_update=lambda e: self._Drag(e, "Right"), on_right_pan_end=lambda e: self._EndDrag(), hover_interval=20, drag_interval=16, expand=True, ) super().__init__( content=ft.Column([ ft.Row([ ft.IconButton(icon=ft.Icons.ARROW_BACK, tooltip="Back to the file list (Esc)", on_click=lambda e: OnClose()), ft.Text("Graph", weight=ft.FontWeight.BOLD), self.ScopePicker, ft.Row(LegendItems, spacing=4, tight=True), ft.Container(expand=True), self.CountText, ], spacing=12), ft.Divider(height=1, color=Palette.Surface1), ft.Stack([self.Surface, self.StatusBox], expand=True), ft.Text(f"{Hint}, scroll to zoom.", size=11, color=Palette.Overlay0), ], spacing=8, expand=True), expand=True, bgcolor=Palette.Base, padding=12, ) def did_mount(self) -> None: self.Reload() def will_unmount(self) -> None: self.Closed = True # --- Data and layout ---------------------------------------------------- def Reload(self) -> None: self.Generation += 1 Generation = self.Generation self.ScopePath = self.DirectoryPath if self.ScopePicker.value == "Directory" else "" self.Status.value = "Building the graph…" self.StatusBox.visible = True self.BaseCanvas.shapes = [] self.LitCanvas.shapes = [] self.Layout = None self.Hovered = None self.update() self.page.run_thread(self._Build, Generation, self.ScopePath) def _Build(self, Generation: int, Path: str) -> None: try: Graph = self.TheApp.Source.Graph(self.Root, Path) except SourceError as Error: if Generation == self.Generation and not self.Closed: self.Status.value = f"Couldn't build the graph: {Error}" self.Status.update() return if Generation != self.Generation or self.Closed: return Directories = sum(1 for Node in Graph["Nodes"] if Node["IsDirectory"]) Files = len(Graph["Nodes"]) - Directories Graph = FilterGraph(Graph, self.ShowDirectories) Count = f"{Directories} directories, {Files} files" if self.ShowDirectories else f"{Files} files" self.CountText.value = Count + (" (cut short: too many to show)" if Graph.get("Truncated") else "") self.CountText.update() if not Graph["Nodes"]: self.Status.value = "Nothing to show here." self.Status.update() return self.Layout = Layout(Graph) Base = len(Path.split("/")) if Path else 0 self.Colours = [DirectoryColour((len(Node["Id"].split("/")) if Node["Id"] else 0) - Base) if Node["IsDirectory"] else Palette.White for Node in self.Layout.Nodes] self.StatusBox.visible = False self.StatusBox.update() # Settle in steps, drawing as it goes, like the web view's animation. while not self.Layout.Settled: for _ in range(TicksPerDraw): self.Layout.Tick() if Generation != self.Generation or self.Closed: return self._Draw() self._Draw() # --- Drawing ------------------------------------------------------------ def _Fit(self) -> None: """Maps the layout onto the canvas, so the whole graph fits at zoom 1.""" Position = self.Layout.Position Low, High = Position.min(0), Position.max(0) Width, Height = self.Size Margin = 40 Scale = min((Width - 2 * Margin) / max(High[0] - Low[0], 1), (Height - 2 * Margin) / max(High[1] - Low[1], 1), 1.5) Scale = max(Scale, 0.05) # the canvas can start out smaller than the margins Middle = (Low + High) / 2 self.Screen = (Position - Middle) * Scale + (Width / 2, Height / 2) Zoom = math.sqrt(Scale) IsDirectory = np.array([Node["IsDirectory"] for Node in self.Layout.Nodes]) Degree = np.sqrt(self.Layout.Degree) self.Radius = np.maximum(2, np.where(IsDirectory, 5 + Degree * 1.6, 3 + Degree * 0.8) * Zoom) def _Label(self, Number: int, Colour: str) -> cv.Text: X, Y = self.Screen[Number] return cv.Text(X, Y + self.Radius[Number] + 2, self.Layout.Nodes[Number]["Name"], style=ft.TextStyle(size=10, color=Colour, font_family=Core.ActiveFontFamily), alignment=ft.Alignment.TOP_CENTER) def _Dot(self, Number: int) -> cv.Circle: X, Y = self.Screen[Number] return cv.Circle(X, Y, self.Radius[Number], paint=ft.Paint(color=self.Colours[Number])) def _Line(self, LinkNumber: int, Colour: str, Width: float = 1) -> cv.Line: A, B = self.Screen[self.Layout.Sources[LinkNumber]], self.Screen[self.Layout.Targets[LinkNumber]] return cv.Line(A[0], A[1], B[0], B[1], paint=ft.Paint(color=Colour, stroke_width=Width, style=ft.PaintingStyle.STROKE)) def _Draw(self) -> None: if self.Layout is None or self.Closed: return self._Fit() Shapes: list = [] for Number, Kind in enumerate(self.Layout.Kinds): Shapes.append(self._Line(Number, Palette.Surface1 if Kind == "Child" else Palette.Blue)) Nodes = self.Layout.Nodes ShowFileLabels = len(Nodes) <= 300 for Number, Node in enumerate(Nodes): Shapes.append(self._Dot(Number)) if Node["IsDirectory"] or ShowFileLabels: Shapes.append(self._Label(Number, Palette.Overlay0)) self.BaseCanvas.shapes = Shapes self.BaseCanvas.update() self._DrawHighlight() def _DrawHighlight(self) -> None: """The hovered dot and its links, over a veil that dims the rest.""" Number = self.Hovered if Number is None or self.Layout is None: self.LitCanvas.shapes = [] else: Width, Height = self.Size Shapes: list = [cv.Rect(-4000, -4000, Width + 8000, Height + 8000, paint=ft.Paint(color=ft.Colors.with_opacity(0.75, Palette.Base)))] Lit = {Number} | self.Layout.Neighbours[Number] for LinkNumber in range(len(self.Layout.Kinds)): if Number in (self.Layout.Sources[LinkNumber], self.Layout.Targets[LinkNumber]): Shapes.append(self._Line(LinkNumber, Palette.Mauve, 1.5)) for Other in Lit: Shapes.append(self._Dot(Other)) Shapes.append(self._Label(Other, Palette.Text if Other == Number else Palette.Overlay0)) X, Y = self.Screen[Number] Shapes.append(cv.Circle(X, Y, self.Radius[Number] + 3, paint=ft.Paint(color=Palette.Mauve, stroke_width=2, style=ft.PaintingStyle.STROKE))) self.LitCanvas.shapes = Shapes self.LitCanvas.update() # --- View transform ----------------------------------------------------- def _ApplyView(self) -> None: self.Plane.transform = ft.Transform( matrix=ft.Matrix4.identity().translate(self.PanX, self.PanY).scale(self.Zoom, self.Zoom), alignment=ft.Alignment.TOP_LEFT, ) self.Plane.update() def _ToCanvas(self, X: float, Y: float) -> tuple[float, float]: return (X - self.PanX) / self.Zoom, (Y - self.PanY) / self.Zoom def _PanBy(self, DX: float, DY: float) -> None: self.PanX += DX self.PanY += DY self._ApplyView() # --- Input -------------------------------------------------------------- def _NodeAt(self, X: float, Y: float) -> Optional[int]: """The dot under a point on the surface (screen coordinates).""" if self.Layout is None or not len(self.Screen): return None X, Y = self._ToCanvas(X, Y) Distance = np.hypot(self.Screen[:, 0] - X, self.Screen[:, 1] - Y) Number = int(Distance.argmin()) Reach = max(6 / self.Zoom, self.Radius[Number] + 4 / self.Zoom) # at least 6 screen pixels return Number if Distance[Number] <= Reach else None def _OnResize(self, Event) -> None: self.Size = (Event.width, Event.height) self._Draw() def _SetHovered(self, Number: Optional[int]) -> None: if Number != self.Hovered: self.Hovered = Number self._DrawHighlight() def _OnHover(self, Event) -> None: X, Y = Event.local_position.x, Event.local_position.y self._SetHovered(self._NodeAt(X, Y)) def _OnClick(self, Event, Button: str) -> None: if Button != self.OpenButton: return Number = self._NodeAt(Event.local_position.x, Event.local_position.y) if Number is not None: self.OnOpenNode(self.Layout.Nodes[Number]) def _StartDrag(self, Event, Button: str) -> None: if Button == self.PanButton: self.DragFrom = (Event.local_position.x, Event.local_position.y) def _Drag(self, Event, Button: str) -> None: if Button != self.PanButton or self.DragFrom is None: return X, Y = Event.local_position.x, Event.local_position.y self._PanBy(X - self.DragFrom[0], Y - self.DragFrom[1]) self.DragFrom = (X, Y) def _EndDrag(self) -> None: self.DragFrom = None def _OnScroll(self, Event) -> None: """Zooms around the pointer.""" X, Y = Event.local_position.x, Event.local_position.y CanvasX, CanvasY = self._ToCanvas(X, Y) self.Zoom = min(8.0, max(0.2, self.Zoom * math.exp(-Event.scroll_delta.y * 0.0015))) self.PanX = X - CanvasX * self.Zoom self.PanY = Y - CanvasY * self.Zoom self._ApplyView()