Make widget state changes reactive

Substate stores

pnpm-lock.yaml

@@ -21,6 +21,7 @@ importers:
       eslint-config-prettier: ^8.8.0
       eslint-plugin-svelte3: ^4.0.0
       events: ^3.3.0
+      immer-loves-svelte: ^2.2.4
       pollen-css: ^4.6.2
       prettier: ^2.8.7
       prettier-plugin-svelte: ^2.10.0
@@ -52,6 +53,7 @@ importers:
       '@litegraph-ts/core': link:litegraph/packages/core
       '@litegraph-ts/nodes-basic': link:litegraph/packages/nodes-basic
       events: 3.3.0
+      immer-loves-svelte: 2.2.4
       pollen-css: 4.6.2
       radix-icons-svelte: 1.2.1
       svelte-preprocess: 5.0.3_3gubijbxbisgisegeglxqngyuq
@@ -1003,6 +1005,19 @@ packages:
     engines: {node: '>= 4'}
     dev: true
 
+  /immer-loves-svelte/2.2.4:
+    resolution: {integrity: sha512-CQ/PS8nymZg/4Jhu2X6qsrUdtjPyPz25S8j9xnsT/J+idPFVvhtRH/+yZKDXleu0Na4ZtfPQoVpWz43HD1XFAQ==}
+    engines: {node: '>=10'}
+    dependencies:
+      immer: 9.0.21
+      svelte: 3.58.0
+      underscore: 1.13.6
+    dev: false
+
+  /immer/9.0.21:
+    resolution: {integrity: sha512-bc4NBHqOqSfRW7POMkHd51LvClaeMXpm8dx0e8oE2GORbq5aRK7Bxl4FyzVLdGtLmvLKL7BTDBG5ACQm4HWjTA==}
+    dev: false
+
   /immutable/4.3.0:
     resolution: {integrity: sha512-0AOCmOip+xgJwEVTQj1EfiDDOkPmuyllDuTuEX+DDXUgapLAsBIfkg3sxCYyCEA8mQqZrrxPUGjcOQ2JS3WLkg==}
 
@@ -1858,6 +1873,10 @@ packages:
     engines: {node: '>=12.20'}
     hasBin: true
 
+  /underscore/1.13.6:
+    resolution: {integrity: sha512-+A5Sja4HP1M08MaXya7p5LvjuM7K6q/2EaC0+iovj/wOcMsTzMvDFbasi/oSapiwOlt252IqsKqPjCl7huKS0A==}
+    dev: false
+
   /undici/5.20.0:
     resolution: {integrity: sha512-J3j60dYzuo6Eevbawwp1sdg16k5Tf768bxYK4TUJRH7cBM4kFCbf3mOnM/0E3vQYXvpxITbbWmBafaDbxLDz3g==}
     engines: {node: '>=12.18'}

src/lib/components/ComfyApp.svelte

@@ -33,9 +33,8 @@
  }
 
  function queuePrompt() {
-     const state = get(widgetState);
+     console.log("Queuing!");
-     console.log("Queuing!", state);
+     app.queuePrompt(0, 1);
-     app.queuePrompt(0, 1, state);
  }
 
  $: if (app) app.lCanvas.allow_dragnodes = !nodesLocked;
@@ -69,28 +68,10 @@
 
  function serializeAppState(): SerializedAppState {
      const graph = app.lGraph;
-     const frontendState = get(widgetState);
 
      const serializedGraph = graph.serialize()
      const serializedPaneOrder = uiPane.serialize()
 
-     // Override the saved graph widget state with the properties set in the
-     // frontend panels.
-     for (let i = 0; i < serializedGraph.nodes.length; i++) {
-         let serializedNode = serializedGraph.nodes[i];
-         let frontendWidgetStates = frontendState[serializedNode.id];
-         if (frontendWidgetStates && serializedNode.widgets_values) {
-             for (let j = 0; j < serializedNode.widgets_values.length; j++) {
-                 let frontendWidgetState = frontendWidgetStates[j];
-
-                 // Virtual widgets always come after real widgets in the current design
-                 if (frontendWidgetState && !frontendWidgetState.isVirtual) {
-                     serializedNode.widgets_values[j] = frontendWidgetState.value;
-                 }
-             }
-         }
-     }
-
      return {
          createdBy: "ComfyBox",
          version: 1,

src/lib/components/ComfyApp.ts

@@ -16,6 +16,7 @@ import type { WidgetStateStore, WidgetUIState } from "$lib/stores/widgetState";
 import * as widgets from "$lib/widgets/index"
 import type ComfyWidget from "$lib/widgets/ComfyWidget";
 import queueState from "$lib/stores/queueState";
+import GraphSync from "$lib/GraphSync";
 
 LiteGraph.catch_exceptions = false;
 
@@ -70,6 +71,7 @@ export default class ComfyApp {
     dropZone: HTMLElement | null = null;
     nodeOutputs: Record<string, any> = {};
     eventBus: TypedEmitter<ComfyAppEvents> = new EventEmitter() as TypedEmitter<ComfyAppEvents>;
+    graphSync: GraphSync;
 
     dragOverNode: LGraphNode | null = null;
     shiftDown: boolean = false;
@@ -88,6 +90,7 @@ export default class ComfyApp {
         this.lGraph = new LGraph();
         this.lCanvas = new ComfyGraphCanvas(this, this.canvasEl, this.lGraph);
         this.canvasCtx = this.canvasEl.getContext("2d");
+        this.graphSync = new GraphSync(this);
 
         this.addGraphLifecycleHooks();
 
@@ -458,14 +461,12 @@ export default class ComfyApp {
      * Converts the current graph workflow for sending to the API
      * @returns The workflow and node links
      */
-    async graphToPrompt(frontendState: WidgetStateStore = {}) {
+    async graphToPrompt() {
         const workflow = this.lGraph.serialize();
 
         const output = {};
         // Process nodes in order of execution
         for (const node of this.lGraph.computeExecutionOrder<ComfyGraphNodeExecutable>(false, null)) {
-            const fromFrontend: WidgetUIState[] | null = frontendState[node.id];
-
             const n = workflow.nodes.find((n) => n.id === node.id);
 
             if (node.isVirtualNode || !node.comfyClass) {
@@ -491,9 +492,6 @@ export default class ComfyApp {
                     const widget = widgets[i];
                     if (!widget.options || widget.options.serialize !== false) {
                         let value = widget.serializeValue ? await widget.serializeValue(n, i) : widget.value;
-                        if (fromFrontend && !fromFrontend[i].isVirtual) {
-                            value = fromFrontend[i].value;
-                        }
                         inputs[widget.name] = value
                     }
                 }
@@ -540,7 +538,7 @@ export default class ComfyApp {
         return { workflow, output };
     }
 
-    async queuePrompt(num: number, batchCount: number = 1, frontendState: WidgetStateStore = {}) {
+    async queuePrompt(num: number, batchCount: number = 1) {
         this.queueItems.push({ num, batchCount });
 
         // Only have one action process the items so each one gets a unique seed correctly
@@ -555,7 +553,7 @@ export default class ComfyApp {
                 console.log(`Queue get! ${num} ${batchCount}`);
 
                 for (let i = 0; i < batchCount; i++) {
-                    const p = await this.graphToPrompt(frontendState);
+                    const p = await this.graphToPrompt();
 
                     try {
                         await this.api.queuePrompt(num, p);

src/lib/components/ComfyUIPane.svelte

@@ -5,7 +5,7 @@
  import ComfyApp  from "./ComfyApp";
  import type { SerializedPanes } from "./ComfyApp"
  import ComfyPane from "./ComfyPane.svelte";
- import widgetState, { type WidgetUIState } from "$lib/stores/widgetState";
+ import widgetState from "$lib/stores/widgetState";
 	import type { DragItem } from "./ComfyUIPane";
 
  export let app: ComfyApp;

src/lib/components/LightboxModal.svelte

@@ -53,6 +53,7 @@
 
  #modalImage {
      overflow: hidden;
+     user-drag: none;
  }
 
  .modalControls {

src/lib/graphSync.ts

@@ -0,0 +1,78 @@
+import type { LGraph } from "@litegraph-ts/core";
+import widgetState, { type WidgetStateStore, type WidgetUIStateStore } from "./stores/widgetState";
+import type ComfyApp from "./components/ComfyApp";
+import type { Unsubscriber } from "svelte/store";
+
+type WidgetSubStore = {
+    store: WidgetUIStateStore,
+    unsubscribe: Unsubscriber
+}
+
+/*
+ * Responsible for watching and synchronizing state changes from the frontend to the litegraph instance.
+ * The other way around is unnecessary since the nodes in ComfyBox can't be interacted with.
+ *
+ * Assumptions:
+ * - Widgets can't be added to a node after they're created
+ * - Widgets can't be interacted with from the graph, only from the frontend
+ */
+export default class GraphSync {
+    graph: LGraph;
+    private _unsubscribe: Unsubscriber;
+    private _finalizer: FinalizationRegistry<number>;
+
+    // nodeId -> widgetSubStores[]
+    private stores: Record<string, WidgetSubStore[]> = {}
+
+    constructor(app: ComfyApp) {
+        this.graph = app.lGraph;
+        this._unsubscribe = widgetState.subscribe(this.onWidgetStateChanged.bind(this));
+        this._finalizer = new FinalizationRegistry((id: number) => {
+            console.log(`${this} has been garbage collected`);
+            this._unsubscribe();
+        });
+    }
+
+    private onWidgetStateChanged(state: WidgetStateStore) {
+        // TODO assumes only a single graph's widget state.
+
+        console.warn("ONWIDGETSTATECHANGE")
+
+        for (let nodeId in state) {
+            if (!this.stores[nodeId]) {
+                this.addStores(state, nodeId);
+            }
+        }
+
+        for (let nodeId in this.stores) {
+            if (!state[nodeId]) {
+                this.removeStores(nodeId);
+            }
+        }
+    }
+
+    private addStores(state: WidgetStateStore, nodeId: string) {
+        if (this.stores[nodeId]) {
+            console.warn("Stores already exist!", nodeId, this.stores[nodeId])
+        }
+
+        this.stores[nodeId] = []
+
+        for (const wuis of state[nodeId]) {
+            const unsub = wuis.value.subscribe((v) => {
+                console.log("CHANGE", v)
+            })
+            this.stores[nodeId].push({ store: wuis.value, unsubscribe: unsub });
+        }
+
+        console.log("NEWSTORES", this.stores[nodeId])
+    }
+
+    private removeStores(nodeId: string) {
+        console.log("DELSTORES", this.stores[nodeId])
+        for (const ss of this.stores[nodeId]) {
+            ss.unsubscribe();
+        }
+        delete this.stores[nodeId]
+    }
+}

src/lib/stores/widgetState.ts

@@ -4,10 +4,15 @@ import type { Readable, Writable } from 'svelte/store';
 import type ComfyGraphNode from '$lib/nodes/ComfyGraphNode';
 import type ComfyWidget from '$lib/widgets/ComfyWidget';
 
+import { subStore } from "immer-loves-svelte"
+
+/** store for one widget's state */
+export type WidgetUIStateStore = Writable<any>
+
 export type WidgetUIState = {
     node: LGraphNode,
     widget: IWidget,
-    value: any,
+    value: WidgetUIStateStore,
     isVirtual: boolean
 }
 
@@ -41,7 +46,7 @@ function nodeAdded(node: LGraphNode) {
         for (const widget of node.widgets) {
             if (!state[node.id])
                 state[node.id] = []
-            state[node.id].push({ node, widget, value: widget.value, isVirtual: false })
+            state[node.id].push({ node, widget, value: writable(widget.value), isVirtual: false })
         }
     }
 
@@ -50,10 +55,12 @@ function nodeAdded(node: LGraphNode) {
         for (const widget of comfyNode.virtualWidgets) {
             if (!state[comfyNode.id])
                 state[comfyNode.id] = []
-            state[comfyNode.id].push({ node, widget, value: widget.value, isVirtual: true })
+            state[comfyNode.id].push({ node, widget, value: writable(widget.value), isVirtual: true })
         }
     }
 
+    console.log("NODEADDED", state)
+
     store.set(state);
 }
 
@@ -69,7 +76,7 @@ function widgetStateChanged(widget: ComfyWidget<any, any>) {
     if (entries) {
         let widgetState = entries.find(e => e.widget === widget);
         if (widgetState) {
-            widgetState.value = widget.value;
+            widgetState.value.set(widget.value);
             store.set(state);
         }
         else {
@@ -88,7 +95,7 @@ function configureFinished(graph: LGraph) {
         if (node.widgets_values) {
             for (const [i, value] of node.widgets_values.entries()) {
                 if (i < state[node.id].length && !state[node.id][i].isVirtual) { // Virtual widgets always come after real widgets
-                    state[node.id][i].value = value;
+                    state[node.id][i].value.set(value);
                 }
                 else {
                     console.error("Mismatch in widgets_values!", state[node.id].map(i => i.value), node.widgets_values)

src/lib/widgets/ComboWidget.svelte

@@ -1,13 +1,17 @@
 <script lang="ts">
- import type { WidgetDrawState, WidgetUIState } from "$lib/stores/widgetState";
+ import type { WidgetDrawState, WidgetUIState, WidgetUIStateStore } from "$lib/stores/widgetState";
  import { BlockTitle } from "@gradio/atoms";
  import { Dropdown } from "@gradio/form";
+ import { get } from "svelte/store";
  import Select from 'svelte-select';
  export let item: WidgetUIState | null = null;
+ let itemValue: WidgetUIStateStore | null = null;
+ let option: any;
 
- let option: any = null;
+ $: if(item) {
-
+     option = get(item.value);
- $: if(item && !option) option = item.value;
+     itemValue = item.value
+ };
 </script>
 
 <div class="wrapper">
@@ -16,7 +20,7 @@
             <BlockTitle show_label={true}>{item.widget.name}</BlockTitle>
             <Select
                 bind:value={option}
-                bind:justValue={item.value}
+                bind:justValue={$itemValue}
                 bind:items={item.widget.options.values}
                 disabled={item.widget.options.values.length === 0}
                 clearable={false}

src/lib/widgets/ComfyGalleryWidget.svelte

@@ -1,11 +1,17 @@
 <script lang="ts">
  import { onMount } from "svelte";
  import { ImageViewer } from "$lib/ImageViewer";
- import type { WidgetUIState } from "$lib/stores/widgetState";
+ import type { WidgetUIState, WidgetUIStateStore } from "$lib/stores/widgetState";
  import { Block } from "@gradio/atoms";
  import { Gallery } from "@gradio/gallery";
  import type { Styles } from "@gradio/utils";
+
  export let item: WidgetUIState | null = null;
+ let itemValue: WidgetUIStateStore | null = null; // stores must be declared at top level
+
+ $: if(item) {
+     itemValue = item.value;
+ }
 
  let style: Styles = {
      // grid_cols: [2],
@@ -28,10 +34,10 @@
 
 </script>
 <div class="wrapper comfy-gallery-widget" bind:this={element}>
-    {#if item}
+    {#if item && itemValue}
         <Block variant="solid" padding={false}>
             <Gallery
-                bind:value={item.value}
+                bind:value={$itemValue}
                 label={item.widget.name}
                 show_label={true}
                 {style}

src/lib/widgets/RangeWidget.svelte

@@ -1,13 +1,15 @@
 <script lang="ts">
- import type { WidgetUIState } from "$lib/stores/widgetState";
+ import type { WidgetUIState, WidgetUIStateStore } from "$lib/stores/widgetState";
  import { Range } from "@gradio/form";
  export let item: WidgetUIState | null = null;
+ let itemValue: WidgetUIStateStore | null = null;
+ $: if (item) { itemValue = item.value; }
 </script>
 
 <div class="wrapper">
-    {#if item}
+    {#if item && itemValue}
         <Range
-            bind:value={item.value}
+            bind:value={$itemValue}
             minimum={item.widget.options.min}
             maximum={item.widget.options.max}
             step={item.widget.options.step}

src/lib/widgets/TextWidget.svelte

@@ -1,13 +1,15 @@
 <script lang="ts">
- import type { WidgetUIState } from "$lib/stores/widgetState";
+ import type { WidgetUIState, WidgetUIStateStore } from "$lib/stores/widgetState";
  import { TextBox } from "@gradio/form";
  export let item: WidgetUIState | null = null;
+ let itemValue: WidgetUIStateStore | null = null;
+ $: if (item) { itemValue = item.value; }
 </script>
 
 <div class="wrapper">
-    {#if item}
+    {#if item && itemValue}
         <TextBox
-            bind:value={item.value}
+            bind:value={$itemValue}
             label={item.widget.name}
             lines={item.widget.options.multiline ? 5 : 1}
             max_lines={item.widget.options.multiline ? 5 : 1}