Prompt travel

src/lib/components/ComfyApp.ts

@@ -40,6 +40,7 @@ import { deserializeTemplateFromSVG, type SerializedComfyBoxTemplate } from "$li
 import templateState from "$lib/stores/templateState";
 import { formatValidationError, type ComfyAPIPromptErrorResponse, formatExecutionError, type ComfyExecutionError } from "$lib/apiErrors";
 import systemState from "$lib/stores/systemState";
+import type { JourneyNode } from "$lib/stores/journeyStates";
 
 export const COMFYBOX_SERIAL_VERSION = 1;
 
@@ -610,6 +611,7 @@ export default class ComfyApp {
                     if (node?.onExecuted) {
                         node.onExecuted(output);
                     }
+                    workflow.journey.onExecuted(promptID, nodeID, output, queueEntry);
                 }
             }
         });
@@ -1028,6 +1030,15 @@ export default class ComfyApp {
             notify("Prompt queued.", { type: "info", showOn: "web" });
         }
 
+        let journeyNode: JourneyNode | null;
+
+        if (get(uiState).autoPushJourney) {
+            const activeNode = targetWorkflow.journey.getActiveNode();
+            if (activeNode != null) {
+                journeyNode = targetWorkflow.journey.pushPatchOntoActive(targetWorkflow, activeNode);
+            }
+        }
+
         this.processingQueue = true;
         let workflow: ComfyBoxWorkflow;
 
@@ -1097,6 +1108,9 @@ export default class ComfyApp {
                         else {
                             queueState.afterQueued(workflow.id, response.promptID, response.number, p.output, extraData)
                             workflowState.afterQueued(workflow.id, response.promptID)
+                            if (journeyNode != null) {
+                                journeyNode.promptID = response.promptID;
+                            }
                         }
                     } catch (err) {
                         errorMes = err?.toString();

src/lib/components/ComfyJourneyView.svelte

@@ -7,12 +7,14 @@
  import type ComfyApp from './ComfyApp';
  import type { ComfyBoxWorkflow } from '$lib/stores/workflowState';
  import workflowState from '$lib/stores/workflowState';
- import { calculateWorkflowParamsPatch, resolvePatch, type JourneyPatchNode, type WritableJourneyStateStore } from '$lib/stores/journeyStates';
+ import uiState from '$lib/stores/uiState';
+ import { calculateWorkflowParamsPatch, resolvePatch, type JourneyPatchNode, type WritableJourneyStateStore, diffParams } from '$lib/stores/journeyStates';
  import JourneyRenderer from './JourneyRenderer.svelte';
  import { Plus } from "svelte-bootstrap-icons";
-	import { getWorkflowRestoreParams, getWorkflowRestoreParamsFromWorkflow } from '$lib/restoreParameters';
+ import { getWorkflowRestoreParams, getWorkflowRestoreParamsFromWorkflow } from '$lib/restoreParameters';
-	import notify from '$lib/notify';
+ import notify from '$lib/notify';
-	import selectionState from '$lib/stores/selectionState';
+ import selectionState from '$lib/stores/selectionState';
+	import { Checkbox } from '@gradio/form';
 
  export let app: ComfyApp;
 
@@ -30,32 +32,7 @@
 
      const workflowParams = getWorkflowRestoreParamsFromWorkflow(workflow)
      const activeNode = journey.getActiveNode();
-
+     journey.pushPatchOntoActive(workflow, activeNode, true)
-     let journeyNode
-
-     if (activeNode == null) {
-         // add root node
-         if ($journey.root != null) {
-             return;
-         }
-         journeyNode = journey.addNode(workflowParams, null);
-         notify("Pushed a new base workflow state.", { type: "info" })
-     }
-     else {
-         // add patch node
-         const patch = calculateWorkflowParamsPatch(activeNode, workflowParams);
-         const patchedCount = Object.keys(patch).length;
-         if (patchedCount === 0) {
-             notify("No changes were made to active parameters yet.", { type: "warning" })
-             return;
-         }
-         journeyNode = journey.addNode(patch, activeNode);
-         notify(`Pushed new state with ${patchedCount} changes.`, { type: "info" })
-     }
-
-     if (journeyNode != null) {
-         journey.selectNode(journeyNode);
-     }
  }
 
  function onSelectNode(e: CustomEvent<{ cyto: cytoscape.Core, node: cytoscape.NodeSingular }>) {
@@ -85,12 +62,11 @@
          return;
      }
 
-     if (journeyNode.type === "patch") {
+     const patch = resolvePatch(journeyNode);
-         $selectionState.currentPatchHoveredNodes = new Set(Object.keys((journeyNode as JourneyPatchNode).patch))
+     const workflowParams = getWorkflowRestoreParamsFromWorkflow(workflow);
-     }
+     const diff = diffParams(patch, workflowParams);
-     else {
+
-         $selectionState.currentPatchHoveredNodes = new Set();
+     $selectionState.currentPatchHoveredNodes = new Set(Object.keys(diff));
-     }
  }
 
  function onHoverNodeOut(e: CustomEvent<{ cyto: cytoscape.Core, node: cytoscape.NodeSingular }>) {
@@ -104,6 +80,9 @@
                      on:hover_node={onHoverNode}
                      on:hover_node_out={onHoverNodeOut}
     />
+    <div class="bottom" style:border-top="1px solid var(--panel-border-color)">
+        <Checkbox label="Auto-Push" disabled={$journey.root == null} bind:value={$uiState.autoPushJourney}/>
+    </div>
     <div class="bottom">
         <button class="mode-button ternary"
                 title={"Add new"}
@@ -119,7 +98,7 @@
 
  .journey-view {
      width: 100%;
-     height: calc(100% - $button-height);
+     height: calc(100% - $button-height * 2);
  }
 
  .bottom {
@@ -127,6 +106,7 @@
      display: flex;
      flex-direction: row;
      color: var(--comfy-accent-soft);
+     justify-content: center;
 
      .mode-button {
          height: 100%;

src/lib/components/JourneyRenderer.svelte

@@ -6,6 +6,7 @@
  import type { NodeDataDefinition, EdgeDataDefinition } from 'cytoscape';
  import { createEventDispatcher } from "svelte";
 	import selectionState from '$lib/stores/selectionState';
+	import uiQueueState from '$lib/stores/uiQueueState';
 
  export let workflow: ComfyBoxWorkflow | null = null
  export let journey: WritableJourneyStateStore | null = null
@@ -52,7 +53,7 @@
              const patchNode = node as JourneyPatchNode;
              nodes.push({
                  id: patchNode.id,
-                 label: "N",
+                 label: "P",
                  selected: node.id === journeyState.activeNodeID,
                  locked: true
 
@@ -101,9 +102,22 @@
      const { cyto } = e.detail;
 
      for (const node of cyto.nodes().components()) {
-         if (node.id() === lastSelected) {
+         const nodeID = node.id()
+         if (nodeID === lastSelected) {
              // why doesn't passing `selected` work in the ctor?
              node.select();
+             cyto.zoom(1.25);
+             cyto.center(node)
+         }
+
+         const journeyNode = $journey.nodesByID[nodeID]
+         if (journeyNode) {
+             if (journeyNode.promptID != null) {
+                 const queueEntry = $uiQueueState.historyUIEntries.find(e => e.entry.promptID === journeyNode.promptID)
+                 if (queueEntry != null && queueEntry.images) {
+                     node.data("bgImage", queueEntry.images[0]);
+                 }
+             }
          }
      }
 
@@ -114,13 +128,6 @@
          .on("select", onNodeSelected)
          .on("mouseover", onNodeHovered)
          .on("mouseout", onNodeHoveredOut)
-
-     const nodes = Array.from(journey.iterateBreadthFirst());
-     if (nodes.length > 0) {
-         const lastNode = nodes[nodes.length - 1]
-         const start = cyto.$(`#${lastNode.id}`)
-         cyto.center(start)
-     }
  }
 </script>
 

src/lib/components/graph/Graph.svelte

@@ -37,7 +37,7 @@
          container: refElement,
          style: GraphStyles,
          wheelSensitivity: 0.1,
-         maxZoom: 1,
+         maxZoom: 3,
          minZoom: 0.5,
      })
 

src/lib/components/graph/GraphStyles.ts

@@ -18,8 +18,9 @@ const styles: Stylesheet[] = [
     {
         selector: "node",
         style: {
-            "width": "50",
+            "width": "100",
-            "height": "50",
+            "height": "100",
+            "shape": "round-rectangle",
             "font-family": "Arial",
             "font-size": "18",
             "font-weight": "normal",
@@ -33,6 +34,16 @@ const styles: Stylesheet[] = [
             "color": "#1d3660"
         }
     },
+    {
+        selector: "node[bgImage]",
+        style: {
+            "label": "",
+            "background-image": "data(bgImage)",
+            "background-image-containment": "over",
+            "background-fit": "cover",
+            "color": "transparent"
+        }
+    },
     {
         selector: "node:selected",
         style: {

src/lib/nodes/widgets/ComfyButtonNode.ts

@@ -11,6 +11,7 @@ export default class ComfyButtonNode extends ComfyWidgetNode<boolean> {
     override properties: ComfyButtonProperties = {
         tags: [],
         defaultValue: false,
+        excludeFromJourney: true,
         param: "bang"
     }
 

src/lib/nodes/widgets/ComfyCheckboxNode.ts

@@ -11,6 +11,7 @@ export default class ComfyCheckboxNode extends ComfyWidgetNode<boolean> {
     override properties: ComfyCheckboxProperties = {
         tags: [],
         defaultValue: false,
+        excludeFromJourney: false,
     }
 
     static slotLayout: SlotLayout = {

src/lib/nodes/widgets/ComfyComboNode.ts

@@ -19,7 +19,8 @@ export default class ComfyComboNode extends ComfyWidgetNode<string> {
         tags: [],
         defaultValue: "A",
         values: ["A", "B", "C", "D"],
-        convertValueToLabelCode: ""
+        convertValueToLabelCode: "",
+        excludeFromJourney: false,
     }
 
     static slotLayout: SlotLayout = {

src/lib/nodes/widgets/ComfyGalleryNode.ts

@@ -18,7 +18,8 @@ export default class ComfyGalleryNode extends ComfyWidgetNode<ComfyBoxImageMetad
         defaultValue: [],
         index: 0,
         updateMode: "replace",
-        autoSelectOnUpdate: true
+        autoSelectOnUpdate: true,
+        excludeFromJourney: true,
     }
 
     static slotLayout: SlotLayout = {
@@ -130,8 +131,6 @@ export default class ComfyGalleryNode extends ComfyWidgetNode<ComfyBoxImageMetad
 
         const meta = parseWhateverIntoImageMetadata(param) || [];
 
-        console.debug("[ComfyGalleryNode] Received output!", param)
-
         if (updateMode === "append") {
             const currentValue = get(this.value)
             if (meta.length > 0 && (selectedIndex != null || this.properties.autoSelectOnUpdate)) {

src/lib/nodes/widgets/ComfyImageUploadNode.ts

@@ -6,7 +6,6 @@ import ImageUploadWidget from "$lib/widgets/ImageUploadWidget.svelte";
 import type { ComfyWidgetProperties } from "./ComfyWidgetNode";
 import ComfyWidgetNode from "./ComfyWidgetNode";
 import { get, writable, type Writable } from "svelte/store";
-import { type LineGroup } from "$lib/components/MaskCanvas.svelte"
 
 export interface ComfyImageUploadNodeProperties extends ComfyWidgetProperties {
     maskCount: number
@@ -16,7 +15,8 @@ export default class ComfyImageUploadNode extends ComfyWidgetNode<ComfyBoxImageM
     properties: ComfyImageUploadNodeProperties = {
         defaultValue: [],
         tags: [],
-        maskCount: 0
+        maskCount: 0,
+        excludeFromJourney: true,
     }
 
     static slotLayout: SlotLayout = {

src/lib/nodes/widgets/ComfyMarkdownNode.ts

@@ -10,6 +10,7 @@ export default class ComfyMarkdownNode extends ComfyWidgetNode<string> {
     override properties: ComfyMarkdownProperties = {
         tags: [],
         defaultValue: false,
+        excludeFromJourney: true,
     }
 
     static slotLayout: SlotLayout = {

src/lib/nodes/widgets/ComfyMultiRegionNode.ts

@@ -30,7 +30,8 @@ export default class ComfyMultiRegionNode extends ComfyWidgetNode<BoundingBox[]>
         canvasWidth: 512,
         canvasHeight: 512,
         canvasImageURL: null,
-        inputType: "size"
+        inputType: "size",
+        excludeFromJourney: false,
     }
 
     static slotLayout: SlotLayout = {

src/lib/nodes/widgets/ComfyNumberNode.ts

@@ -20,7 +20,8 @@ export default class ComfyNumberNode extends ComfyWidgetNode<number> {
         min: 0,
         max: 10,
         step: 1,
-        precision: 1
+        precision: 1,
+        excludeFromJourney: false,
     }
 
     override svelteComponentType = NumberWidget

src/lib/nodes/widgets/ComfyRadioNode.ts

@@ -16,6 +16,7 @@ export default class ComfyRadioNode extends ComfyWidgetNode<string> {
         tags: [],
         choices: ["Choice A", "Choice B", "Choice C"],
         defaultValue: "Choice A",
+        excludeFromJourney: false,
     }
 
     static slotLayout: SlotLayout = {

src/lib/nodes/widgets/ComfyTextNode.ts

@@ -16,6 +16,7 @@ export default class ComfyTextNode extends ComfyWidgetNode<string> {
         multiline: false,
         lines: 5,
         maxLines: 5,
+        excludeFromJourney: false,
     }
 
     static slotLayout: SlotLayout = {

src/lib/nodes/widgets/ComfyWidgetNode.ts

@@ -37,7 +37,8 @@ export type SerializedComfyWidgetNode = {
  */
 
 export interface ComfyWidgetProperties extends ComfyGraphNodeProperties {
-    defaultValue: any
+    defaultValue: any,
+    excludeFromJourney: boolean
 }
 
 export type ShownOutputProperty = {

src/lib/restoreParameters.ts

@@ -162,13 +162,16 @@ export function concatRestoreParams2(a: RestoreParamTargets, b: RestoreParamTarg
 /*
  * Like getWorkflowRestoreParams but applies to an instanced (non-serialized) workflow
  */
-export function getWorkflowRestoreParamsFromWorkflow(workflow: ComfyBoxWorkflow): RestoreParamWorkflowNodeTargets {
+export function getWorkflowRestoreParamsFromWorkflow(workflow: ComfyBoxWorkflow, noExclude: boolean = false): RestoreParamWorkflowNodeTargets {
     const result = {}
 
     for (const node of workflow.graph.iterateNodesInOrderRecursive()) {
         if (!isComfyWidgetNode(node))
             continue;
 
+        if (!noExclude && node.properties.excludeFromJourney)
+            continue;
+
         const finalValue = node.getValue();
         if (finalValue != null) {
             const source: RestoreParamSourceWorkflowNode = {

src/lib/stores/journeyStates.ts

@@ -1,11 +1,16 @@
 import { get, writable } from 'svelte/store';
 import type { Readable, Writable } from 'svelte/store';
-import type { DragItemID, IDragItem } from './layoutStates';
+import { isComfyWidgetNode, type DragItemID, type IDragItem } from './layoutStates';
 import { LiteGraph, type LGraphNode, type NodeID, type UUID } from '@litegraph-ts/core';
 import type { SerializedAppState } from '$lib/components/ComfyApp';
-import type { RestoreParamTargets, RestoreParamWorkflowNodeTargets } from '$lib/restoreParameters';
+import { getWorkflowRestoreParamsFromWorkflow, type RestoreParamSourceWorkflowNode, type RestoreParamTargets, type RestoreParamWorkflowNodeTargets } from '$lib/restoreParameters';
 import { v4 as uuidv4 } from "uuid";
 import deepEqual from "deep-equal";
+import notify from '$lib/notify';
+import type { ComfyBoxWorkflow } from './workflowState';
+import type { ComfyNodeID, PromptID } from '$lib/api';
+import type { SerializedPromptOutput } from '$lib/utils';
+import type { QueueEntry } from './queueState';
 
 export type JourneyNodeType = "root" | "patch";
 
@@ -14,7 +19,9 @@ export type JourneyNodeID = UUID;
 export interface JourneyNode {
     id: JourneyNodeID,
     type: JourneyNodeType,
-    children: JourneyPatchNode[]
+    children: JourneyPatchNode[],
+    promptID?: PromptID,
+    images?: string[]
 }
 
 export interface JourneyRootNode extends JourneyNode {
@@ -52,7 +59,7 @@ export function resolvePatch(node: JourneyNode): RestoreParamWorkflowNodeTargets
     return base;
 }
 
-function diffParams(base: RestoreParamWorkflowNodeTargets, updated: RestoreParamWorkflowNodeTargets): RestoreParamWorkflowNodeTargets {
+export function diffParams(base: RestoreParamWorkflowNodeTargets, updated: RestoreParamWorkflowNodeTargets): RestoreParamWorkflowNodeTargets {
     const result = {}
 
     for (const [k, v] of Object.entries(updated)) {
@@ -89,9 +96,11 @@ export type JourneyState = {
 type JourneyStateOps = {
     clear: () => void,
     getActiveNode: () => JourneyNode | null,
-    addNode: (params: RestoreParamWorkflowNodeTargets, parent?: JourneyNodeID | JourneyNode) => JourneyNode,
+    // addNode: (params: RestoreParamWorkflowNodeTargets, parent?: JourneyNodeID | JourneyNode) => JourneyNode,
     selectNode: (id?: JourneyNodeID | JourneyNode) => void,
-    iterateBreadthFirst: (id?: JourneyNodeID | null) => Iterable<JourneyNode>
+    iterateBreadthFirst: (id?: JourneyNodeID | null) => Iterable<JourneyNode>,
+    pushPatchOntoActive: (workflow: ComfyBoxWorkflow, activeNode?: JourneyNode, showNotification?: boolean) => JourneyNode | null
+    onExecuted: (promptID: PromptID, nodeID: ComfyNodeID, output: SerializedPromptOutput, queueEntry: QueueEntry) => void
 }
 
 export type WritableJourneyStateStore = Writable<JourneyState> & JourneyStateOps;
@@ -131,6 +140,7 @@ function create() {
      */
     function addNode(params: RestoreParamWorkflowNodeTargets, parent?: JourneyNodeID | JourneyNode): JourneyNode {
         let _node: JourneyRootNode | JourneyPatchNode;
+
         store.update(s => {
             let parentNode: JourneyNode | null = null
             if (parent != null) {
@@ -167,6 +177,41 @@ function create() {
         return _node;
     }
 
+    function pushPatchOntoActive(workflow: ComfyBoxWorkflow, activeNode?: JourneyNode, showNotification: boolean = false): JourneyNode | null {
+        const workflowParams = getWorkflowRestoreParamsFromWorkflow(workflow)
+
+        let journeyNode
+
+        if (activeNode == null) {
+            // add root node
+            if (get(store).root != null) {
+                return;
+            }
+            journeyNode = addNode(workflowParams, null);
+            if (showNotification)
+                notify("Pushed a new base workflow state.", { type: "info" })
+        }
+        else {
+            // add patch node
+            const patch = calculateWorkflowParamsPatch(activeNode, workflowParams);
+            const patchedCount = Object.keys(patch).length;
+            if (patchedCount === 0) {
+                if (showNotification)
+                    notify("No changes were made to active parameters yet.", { type: "warning" })
+                return;
+            }
+            journeyNode = addNode(patch, activeNode);
+            if (showNotification)
+                notify(`Pushed new state with ${patchedCount} changes.`, { type: "info" })
+        }
+
+        if (journeyNode != null) {
+            selectNode(journeyNode);
+        }
+
+        return journeyNode;
+    }
+
     function selectNode(obj?: JourneyNodeID | JourneyNode) {
         store.update(s => {
             if (typeof obj === "string")
@@ -217,13 +262,23 @@ function create() {
         }
     }
 
+    function onExecuted(promptID: PromptID, nodeID: ComfyNodeID, output: SerializedPromptOutput, queueEntry: QueueEntry) {
+        const journeyNode = Array.from(iterateBreadthFirst()).find(j => j.promptID === promptID);
+        if (journeyNode === null)
+            return;
+
+        // TODO
+    }
+
     return {
         ...store,
         getActiveNode,
         clear,
-        addNode,
+        // addNode,
+        pushPatchOntoActive,
         selectNode,
-        iterateBreadthFirst
+        iterateBreadthFirst,
+        onExecuted
     }
 }
 

src/lib/stores/layoutStates.ts

@@ -512,6 +512,14 @@ const ALL_ATTRIBUTES: AttributesSpecList = [
                 serialize: serializeStringArray,
                 deserialize: deserializeStringArray
             },
+            {
+                name: "excludeFromJourney",
+                type: "boolean",
+                location: "nodeProps",
+                editable: true,
+                defaultValue: false,
+                canShow: isComfyWidgetNode
+            },
 
             // Container tags are contained in the widget attributes
             {

src/lib/stores/uiState.ts

@@ -15,6 +15,8 @@ export type UIState = {
     forceSaveUserState: boolean | null,
 
     activeError: PromptID | null
+
+    autoPushJourney: boolean
 }
 
 type UIStateOps = {
@@ -34,7 +36,9 @@ const store: Writable<UIState> = writable(
         reconnecting: false,
         forceSaveUserState: null,
 
-        activeError: null
+        activeError: null,
+
+        autoPushJourney: true
     })
 
 function reconnecting() {

src/lib/stores/workflowState.ts

@@ -224,7 +224,7 @@ export class ComfyBoxWorkflow {
                 console.error("[applyParamsPatch] Node was not ComfyWidgetNode!!", nodeId, source)
                 continue;
             }
-            node.setValue(source.finalValue);
+            node.value.set(source.finalValue);
         }
     }
 

src/tests/stores/journeyStatesTests.ts

@@ -68,4 +68,32 @@ export default class journeyStateTests extends UnitTest {
             }
         })
     }
+
+    test__patches_exclusions() {
+        const [workflow, layoutState] = ComfyBoxWorkflow.create()
+        const { graph, journey } = workflow;
+        layoutState.initDefaultLayout() // adds 3 containers
+
+        const widget1 = LiteGraph.createNode(ComfyNumberNode);
+        const widget2 = LiteGraph.createNode(ComfyNumberNode);
+        const watch1 = LiteGraph.createNode(Watch);
+        const watch2 = LiteGraph.createNode(Watch);
+
+        graph.add(widget1)
+        graph.add(watch1)
+
+        widget1.properties.excludeFromJourney = true;
+        widget1.connect(0, watch1, 0);
+        widget1.setValue(0)
+
+        let workflowParams = getWorkflowRestoreParamsFromWorkflow(workflow)
+        const root = journey.addNode(workflowParams, null);
+
+        expect(root).toEqual({
+            id: root.id,
+            type: "root",
+            children: [],
+            base: {}
+        });
+    }
 }