Serialize node links instead of widget values

Syncing litegraph widget state is kinda annoying, and unnecessary since
everything will be moved to separate UI component nodes. Instead I
modified the input slot type to store the min/max/step to be copied into
the default UI node later. Now nothing uses litegraph's widgets anymore
This commit is contained in:
space-nuko
2023-04-30 16:37:15 -07:00
parent 1b64c3a502
commit 7880c68d7f
10 changed files with 254 additions and 213 deletions

View File

@@ -0,0 +1,16 @@
import type { INodeInputSlot } from "@litegraph-ts/core";
export type ComfyInputConfig = {
min?: number,
max?: number,
step?: number,
precision?: number,
defaultValue?: any,
values?: any[],
multiline?: boolean
}
export default interface IComfyInputSlot extends INodeInputSlot {
serialize: boolean
config: ComfyInputConfig // stores range min/max/step, etc.
}

View File

@@ -110,10 +110,10 @@
app.eventBus.on("configured", nodeState.configureFinished);
app.eventBus.on("cleared", nodeState.clear);
app.eventBus.on("nodeAdded", layoutState.nodeAdded);
app.eventBus.on("nodeRemoved", layoutState.nodeRemoved);
app.eventBus.on("configured", layoutState.configureFinished);
app.eventBus.on("cleared", layoutState.clear);
// app.eventBus.on("nodeAdded", layoutState.nodeAdded);
// app.eventBus.on("nodeRemoved", layoutState.nodeRemoved);
// app.eventBus.on("configured", layoutState.configureFinished);
// app.eventBus.on("cleared", layoutState.clear);
app.eventBus.on("autosave", doAutosave);
app.eventBus.on("restored", doRestore);

View File

@@ -1,4 +1,4 @@
import { LiteGraph, LGraph, LGraphCanvas, LGraphNode, type LGraphNodeConstructor, type LGraphNodeExecutable, type SerializedLGraph, type SerializedLGraphGroup, type SerializedLGraphNode, type SerializedLLink } from "@litegraph-ts/core";
import { LiteGraph, LGraph, LGraphCanvas, LGraphNode, type LGraphNodeConstructor, type LGraphNodeExecutable, type SerializedLGraph, type SerializedLGraphGroup, type SerializedLGraphNode, type SerializedLLink, NodeMode } from "@litegraph-ts/core";
import type { LConnectionKind, INodeSlot } from "@litegraph-ts/core";
import ComfyAPI from "$lib/api"
import { ComfyWidgets } from "$lib/widgets"
@@ -8,7 +8,7 @@ import EventEmitter from "events";
import type TypedEmitter from "typed-emitter";
// Import nodes
import * as basic from "@litegraph-ts/nodes-basic"
import "@litegraph-ts/nodes-basic"
import * as nodes from "$lib/nodes/index"
import ComfyGraphCanvas from "$lib/ComfyGraphCanvas";
import type ComfyGraphNode from "$lib/nodes/ComfyGraphNode";
@@ -16,6 +16,8 @@ import * as widgets from "$lib/widgets/index"
import type ComfyWidget from "$lib/widgets/ComfyWidget";
import queueState from "$lib/stores/queueState";
import GraphSync from "$lib/GraphSync";
import type { SvelteComponentDev } from "svelte/internal";
import type IComfyInputSlot from "$lib/IComfyInputSlot";
LiteGraph.catch_exceptions = false;
@@ -49,12 +51,6 @@ type ComfyAppEvents = {
restored: (workflow: SerializedAppState) => void
}
interface ComfyGraphNodeExecutable extends LGraphNodeExecutable {
comfyClass: string
isVirtualNode?: boolean;
applyToGraph(workflow: SerializedLGraph<SerializedLGraphNode<LGraphNode>, SerializedLLink, SerializedLGraphGroup>): void;
}
export type Progress = {
value: number,
max: number
@@ -199,7 +195,7 @@ export default class ComfyApp {
}
static node_type_overrides: Record<string, typeof ComfyGraphNode> = {}
static widget_type_overrides: Record<string, Function> = {}
static widget_type_overrides: Record<string, typeof SvelteComponentDev> = {}
private registerNodeTypeOverrides() {
ComfyApp.node_type_overrides["SaveImage"] = nodes.ComfySaveImageNode;
@@ -239,6 +235,7 @@ export default class ComfyApp {
super(title);
this.type = nodeId; // XXX: workaround dependency in LGraphNode.addInput()
(this as any).comfyClass = nodeId;
(this as any).isBackendNode = true;
var inputs = nodeData["input"]["required"];
if (nodeData["input"]["optional"] != undefined) {
inputs = Object.assign({}, nodeData["input"]["required"], nodeData["input"]["optional"])
@@ -261,8 +258,8 @@ export default class ComfyApp {
// Standard type widgets
Object.assign(config, widgets[type](this, inputName, inputData, app) || {});
} else {
// Node connection inputs
this.addInput(inputName, type);
// Node connection inputs (backend)
this.addInput(inputName, type, { color_off: "orange", color_on: "orange" });
}
}
}
@@ -277,7 +274,7 @@ export default class ComfyApp {
s[0] = Math.max(config.minWidth, s[0] * 1.5);
s[1] = Math.max(config.minHeight, s[1]);
this.size = s;
this.serialize_widgets = true;
this.serialize_widgets = false;
// app.#invokeExtensionsAsync("nodeCreated", this);
@@ -428,7 +425,7 @@ export default class ComfyApp {
this.clean();
if (!graphData) {
graphData = structuredClone(defaultGraph.workflow)
// graphData = structuredClone(defaultGraph.workflow)
}
// Patch T2IAdapterLoader to ControlNetLoader since they are the same node now
@@ -443,21 +440,6 @@ export default class ComfyApp {
size[0] = Math.max(node.size[0], size[0]);
size[1] = Math.max(node.size[1], size[1]);
node.size = size;
if (node.widgets) {
// If you break something in the backend and want to patch workflows in the frontend
// This is the place to do this
for (let widget of node.widgets) {
if (node.type == "KSampler" || node.type == "KSamplerAdvanced") {
if (widget.name == "sampler_name") {
if (widget.value.constructor === String && widget.value.startsWith("sample_")) {
widget.value = widget.value.slice(7);
}
}
}
}
}
// this.#invokeExtensions("loadedGraphNode", node);
}
}
@@ -467,60 +449,82 @@ export default class ComfyApp {
* @returns The workflow and node links
*/
async graphToPrompt() {
// Run frontend-only logic
this.lGraph.runStep(1)
const workflow = this.lGraph.serialize();
const output = {};
// Process nodes in order of execution
for (const node of this.lGraph.computeExecutionOrder<ComfyGraphNodeExecutable>(false, null)) {
for (const node of this.lGraph.computeExecutionOrder<ComfyGraphNode>(false, null)) {
const n = workflow.nodes.find((n) => n.id === node.id);
if (node.isVirtualNode || !node.comfyClass) {
if (!node.isBackendNode) {
console.debug("Not serializing node: ", node.type)
// Don't serialize frontend only nodes but let them make changes
if (node.applyToGraph) {
node.applyToGraph(workflow);
}
continue;
}
if (node.mode === 2) {
if (node.mode === NodeMode.NEVER) {
// Don't serialize muted nodes
continue;
}
const inputs = {};
const widgets = node.widgets;
// Store all widget values
if (widgets) {
for (let i = 0; i < widgets.length; i++) {
const widget = widgets[i];
let isVirtual = false;
if ("isVirtual" in widget)
isVirtual = (widget as ComfyWidget<any, any>).isVirtual;
if ((!widget.options || widget.options.serialize !== false) && !isVirtual) {
let value = widget.serializeValue ? await widget.serializeValue(n, i) : widget.value;
inputs[widget.name] = value
// Store all link values
if (node.inputs) {
for (let i = 0; i < node.inputs.length; i++) {
const inp = node.inputs[i];
const inputLink = node.getInputLink(i)
const inputNode = node.getInputNode(i)
if (!inputLink || !inputNode)
continue;
let serialize = true;
if ("config" in inp)
serialize = (inp as IComfyInputSlot).serialize
let isBackendNode = false;
let isInputBackendNode = false;
if ("isBackendNode" in node)
isBackendNode = (node as ComfyGraphNode).isBackendNode;
if ("isBackendNode" in inputNode)
isInputBackendNode = (inputNode as ComfyGraphNode).isBackendNode;
// The reasoning behind this check:
// We only want to serialize inputs to nodes with backend equivalents.
// And in ComfyBox, the nodes in litegraph *never* have widgets, instead they're all inputs.
// All values are passed by separate frontend-only nodes,
// either UI-bound or something like ConstantInteger.
// So we know that any value passed into a backend node *must* come from
// a frontend node.
// The rest (links between backend nodes) will be serialized after this bit runs.
if (serialize && isBackendNode && !isInputBackendNode) {
inputs[inp.name] = inputLink.data
}
}
}
// Store all node links
// Store all links between nodes
for (let i = 0; i < node.inputs.length; i++) {
let parent: ComfyGraphNodeExecutable = node.getInputNode(i) as ComfyGraphNodeExecutable;
let parent: ComfyGraphNode = node.getInputNode(i) as ComfyGraphNode;
if (parent) {
let link = node.getInputLink(i);
while (parent && parent.isVirtualNode) {
while (parent && !parent.isBackendNode) {
link = parent.getInputLink(link.origin_slot);
if (link) {
parent = parent.getInputNode(link.origin_slot) as ComfyGraphNodeExecutable;
parent = parent.getInputNode(link.origin_slot) as ComfyGraphNode;
} else {
parent = null;
}
}
if (link) {
inputs[node.inputs[i].name] = [String(link.origin_id), link.origin_slot];
const input = node.inputs[i]
// TODO can null be a legitimate value in some cases?
// Nodes like CLIPLoader will never have a value in the frontend, hence "null".
if (!(input.name in inputs))
inputs[input.name] = [String(link.origin_id), link.origin_slot];
}
}
}
@@ -573,14 +577,8 @@ export default class ComfyApp {
for (const n of p.workflow.nodes) {
const node = this.lGraph.getNodeById(n.id);
if (node.widgets) {
for (const widget of node.widgets) {
// Allow widgets to run callbacks after a prompt has been queued
// e.g. random seed after every gen
if ("afterQueued" in widget) {
(widget as ComfyWidget<any, any>).afterQueued();
}
}
if ("afterQueued" in node) {
(node as ComfyGraphNode).afterQueued();
}
}
@@ -637,17 +635,18 @@ export default class ComfyApp {
const def = defs[node.type];
for (const widgetNum in node.widgets) {
const widget = node.widgets[widgetNum]
throw "refreshComboInNodes unimplemented!"
// for (const widgetNum in node.widgets) {
// const widget = node.widgets[widgetNum]
if (widget.type == "combo" && def["input"]["required"][widget.name] !== undefined) {
widget.options.values = def["input"]["required"][widget.name][0];
// if (widget.type == "combo" && def["input"]["required"][widget.name] !== undefined) {
// widget.options.values = def["input"]["required"][widget.name][0];
if (!widget.options.values.includes(widget.value)) {
widget.value = widget.options.values[0];
}
}
}
// if (!widget.options.values.includes(widget.value)) {
// widget.value = widget.options.values[0];
// }
// }
// }
}
}

View File

@@ -1,8 +1,9 @@
import type ComfyWidget from "$lib/components/widgets/ComfyWidget";
import { LGraphNode } from "@litegraph-ts/core";
import { LGraph, LGraphNode } from "@litegraph-ts/core";
export default class ComfyGraphNode extends LGraphNode {
isVirtualNode: boolean = false;
comfyClass: string | null
isBackendNode?: boolean;
afterQueued?(): void;
onExecuted?(output: any): void;

View File

@@ -18,9 +18,6 @@ export default class ComfyReroute extends ComfyGraphNode {
}
}
// This node is purely frontend and does not impact the resulting prompt so should not be serialized
override isVirtualNode: boolean = true;
override titleMode: TitleMode = TitleMode.NO_TITLE;
override collapsable: boolean = false;

View File

@@ -0,0 +1,103 @@
import { LiteGraph, type ContextMenuItem, type LGraphNode, type Vector2, LConnectionKind, LLink, LGraphCanvas, type SlotType, TitleMode, type SlotLayout, LGraph, type INodeInputSlot } from "@litegraph-ts/core";
import ComfyGraphNode from "./ComfyGraphNode";
import ComboWidget from "$lib/widgets/ComboWidget.svelte";
import RangeWidget from "$lib/widgets/RangeWidget.svelte";
import TextWidget from "$lib/widgets/TextWidget.svelte";
import type { SvelteComponentDev } from "svelte/internal";
import { ComfyWidgets } from "$lib/widgets";
export interface ComfyWidgetProperties extends Record<string, any> {
value: any,
}
/*
* A node that is tied to a UI widget in the frontend. When the frontend's
* widget is changed, the value of the first output in the node is updated
* in the litegraph instance.
*/
export abstract class ComfyWidgetNode extends ComfyGraphNode {
abstract properties: ComfyWidgetProperties;
/** Svelte class for the frontend logic */
abstract svelteComponentType: typeof SvelteComponentDev
/** Compatible litegraph widget types that can be connected to this node */
abstract inputWidgetTypes: string[]
override isBackendNode = false;
override serialize_widgets = true;
override onExecute() {
// Assumption: we will have one output in the inherited class with the
// correct type
this.setOutputData(0, this.properties.value)
// TODO send event to linked nodes
}
}
export interface ComfySliderProperties extends ComfyWidgetProperties {
value: number
}
export class ComfySliderNode extends ComfyWidgetNode {
override properties: ComfySliderProperties = {
value: 0
}
override svelteComponentType = RangeWidget
override inputWidgetTypes = ["number", "slider"]
constructor(name?: string) {
super(name)
this.addOutput("value", "number");
}
}
LiteGraph.registerNodeType({
class: ComfySliderNode,
title: "ComfyBox.UI.Slider",
desc: "Slider outputting a number value",
type: "comfybox/ui/slider"
})
export interface ComfyComboProperties extends ComfyWidgetProperties {
options: string[],
value: string
}
export class ComfyComboNode extends ComfyWidgetNode {
override properties: ComfyComboProperties = {
options: ["*"],
value: "*"
}
override svelteComponentType = ComboWidget
override inputWidgetTypes = ["combo", "enum"]
}
LiteGraph.registerNodeType({
class: ComfyComboNode,
title: "ComfyBox.UI.Combo",
desc: "Combo box outputting a string value",
type: "comfybox/ui/combo"
})
export interface ComfyTextProperties extends ComfyWidgetProperties {
value: string
}
export class ComfyTextNode extends ComfyWidgetNode {
override properties: ComfyTextProperties = {
value: ""
}
override svelteComponentType = TextWidget
override inputWidgetTypes = ["text"]
}
LiteGraph.registerNodeType({
class: ComfyTextNode,
title: "ComfyBox.UI.Text",
desc: "Textbox outputting a string value",
type: "comfybox/ui/text"
})

View File

@@ -221,12 +221,12 @@ function nodeAdded(node: LGraphNode) {
const parent = findDefaultContainerForInsertion();
// Add default node panel containing all widgets
if (node.widgets && node.widgets.length > 0) {
const container = addContainer(parent.id, { title: node.title, direction: "vertical", associatedNode: node.id });
for (const widget of node.widgets) {
addWidget(container.id, node, widget, { associatedNode: node.id });
}
}
// if (node.widgets && node.widgets.length > 0) {
// const container = addContainer(parent.id, { title: node.title, direction: "vertical", associatedNode: node.id });
// for (const widget of node.widgets) {
// addWidget(container.id, node, widget, { associatedNode: node.id });
// }
// }
}
function removeEntry(state: LayoutState, id: DragItemID) {

View File

@@ -63,14 +63,14 @@ function nodeAdded(node: LGraphNode) {
const widgets = [];
if (node.widgets) {
for (const [index, widget] of node.widgets.entries()) {
let isVirtual = false;
if ("isVirtual" in widget)
isVirtual = (widget as ComfyWidget<any, any>).isVirtual;
widgets.push({ index, widget, value: writable(widget.value), isVirtual: isVirtual })
}
}
// if (node.widgets) {
// for (const [index, widget] of node.widgets.entries()) {
// let isVirtual = false;
// if ("isVirtual" in widget)
// isVirtual = (widget as ComfyWidget<any, any>).isVirtual;
// widgets.push({ index, widget, value: writable(widget.value), isVirtual: isVirtual })
// }
// }
state[node.id] = { node: node, name: node.title, widgetStates: widgets }
store.set(state);
@@ -97,17 +97,17 @@ function configureFinished(graph: LGraph) {
for (const node of graph.computeExecutionOrder(false, null)) {
state[node.id].name = node.title;
const widgetStates = state[node.id].widgetStates;
if (node.widgets_values) {
for (const [i, value] of node.widgets_values.entries()) {
if (i < widgetStates.length && !widgetStates[i].isVirtual) {
widgetStates[i].value.set(value);
}
else {
console.log("Skip virtual widget", node.id, node.type, widgetStates[i].widget)
}
}
}
// const widgetStates = state[node.id].widgetStates;
// if (node.widgets_values) {
// for (const [i, value] of node.widgets_values.entries()) {
// if (i < widgetStates.length && !widgetStates[i].isVirtual) {
// widgetStates[i].value.set(value);
// }
// else {
// console.log("Skip virtual widget", node.id, node.type, widgetStates[i].widget)
// }
// }
// }
}
store.set(state)

View File

@@ -5,6 +5,7 @@ import TextWidget from "$lib/widgets/TextWidget.svelte";
import { type WidgetUIState } from "$lib/stores/nodeState";
import { get } from "svelte/store"
import layoutState from "$lib/stores/layoutState"
import type { SvelteComponentDev } from "svelte/internal";
export function download(filename: string, text: string, type: string = "text/plain") {
const blob = new Blob([text], { type: type });
@@ -20,13 +21,11 @@ export function download(filename: string, text: string, type: string = "text/pl
}, 0);
}
export function getComponentForWidgetState(item: WidgetUIState): any {
let ctor: any = null;
export function getComponentForWidgetState(item: WidgetUIState): typeof SvelteComponentDev {
// custom widgets with TypeScript sources
let override = ComfyApp.widget_type_overrides[item.widget.type]
if (override) {
return override;
return override
}
// litegraph.ts built-in widgets

View File

@@ -1,144 +1,70 @@
import type { IWidget, LGraphNode } from "@litegraph-js/core";
import type ComfyApp from "$lib/components/ComfyApp";
import ComfyValueControlWidget from "./widgets/ComfyValueControlWidget";
import type { ComfyInputConfig } from "./IComfyInputSlot";
import type IComfyInputSlot from "./IComfyInputSlot";
import { BuiltInSlotShape } from "@litegraph-ts/core";
export interface WidgetData {
widget: IWidget,
minWidth?: number,
minHeight?: number
}
type WidgetFactory = (node: LGraphNode, inputName: string, inputData: any, app: ComfyApp) => IComfyInputSlot;
type WidgetFactory = (node: LGraphNode, inputName: string, inputData: any, app: ComfyApp) => WidgetData;
type NumberConfig = { min: number, max: number, step: number, precision: number }
type NumberDefaults = { val: number, config: NumberConfig }
function getNumberDefaults(inputData: any, defaultStep: number): NumberDefaults {
let defaultVal = inputData[1]["default"];
function getNumberDefaults(inputData: any, defaultStep: number): ComfyInputConfig {
let defaultValue = inputData[1]["default"];
let { min, max, step } = inputData[1];
if (defaultVal == undefined) defaultVal = 0;
if (defaultValue == undefined) defaultValue = 0;
if (min == undefined) min = 0;
if (max == undefined) max = 2048;
if (step == undefined) step = defaultStep;
return { val: defaultVal, config: { min, max, step: step, precision: 0 } };
return { min, max, step: step, precision: 0, defaultValue };
}
const FLOAT: WidgetFactory = (node: LGraphNode, inputName: string, inputData: any): WidgetData => {
const { val, config } = getNumberDefaults(inputData, 0.5);
return { widget: node.addWidget("number", inputName, val, () => { }, config) };
function addComfyInput(node: LGraphNode, inputName: string, extraInfo: Partial<IComfyInputSlot> = {}): IComfyInputSlot {
const input = node.addInput(inputName) as IComfyInputSlot
for (const [k, v] of Object.entries(extraInfo))
input[k] = v
input.serialize = true;
input.shape = BuiltInSlotShape.CARD_SHAPE;
input.color_off = "lightblue"
input.color_on = "lightblue"
return input;
}
const FLOAT: WidgetFactory = (node: LGraphNode, inputName: string, inputData: any): IComfyInputSlot => {
const config = getNumberDefaults(inputData, 0.5);
return addComfyInput(node, inputName, { type: "number", config })
}
const INT: WidgetFactory = (node: LGraphNode, inputName: string, inputData: any): WidgetData => {
const { val, config } = getNumberDefaults(inputData, 1);
return {
widget: node.addWidget(
"number",
inputName,
val,
function(v) {
const s = this.options.step;
this.value = Math.round(v / s) * s;
},
config
),
};
const INT: WidgetFactory = (node: LGraphNode, inputName: string, inputData: any): IComfyInputSlot => {
const config = getNumberDefaults(inputData, 1);
return addComfyInput(node, inputName, { type: "number", config })
};
function seedWidget(node, inputName, inputData, app) {
const seed = INT(node, inputName, inputData, app);
const seedControl = new ComfyValueControlWidget("control_after_generate", "randomize", node, seed.widget);
node.addCustomWidget(seedControl);
// seed.widget.linkedWidgets = [seedControl];
return seed;
}
const STRING: WidgetFactory = (node: LGraphNode, inputName: string, inputData: any, app: ComfyApp): WidgetData => {
const defaultVal = inputData[1].default || "";
const STRING: WidgetFactory = (node: LGraphNode, inputName: string, inputData: any, app: ComfyApp): IComfyInputSlot => {
const defaultValue = inputData[1].default || "";
const multiline = !!inputData[1].multiline;
// if (multiline) {
// return addMultilineWidget(node, inputName, { defaultVal, ...inputData[1] }, app);
// } else {
return { widget: node.addWidget("text", inputName, defaultVal, () => { }, { multiline }) };
// }
return addComfyInput(node, inputName, { type: "string", config: { defaultValue, multiline } })
};
const COMBO: WidgetFactory = (node: LGraphNode, inputName: string, inputData: any): WidgetData => {
const COMBO: WidgetFactory = (node: LGraphNode, inputName: string, inputData: any): IComfyInputSlot => {
const type = inputData[0];
let defaultValue = type[0];
if (inputData[1] && inputData[1].default) {
defaultValue = inputData[1].default;
}
return { widget: node.addWidget("combo", inputName, defaultValue, () => { }, { values: type }) };
return addComfyInput(node, inputName, { type: "string", config: { values: type, defaultValue } })
}
const IMAGEUPLOAD: WidgetFactory = (node: LGraphNode, inputName: string, inputData: any, app): WidgetData => {
const imageWidget = node.widgets.find((w) => w.name === "image");
let uploadWidget: IWidget;
// async function uploadFile(file: File, updateNode: boolean) {
// try {
// // Wrap file in formdata so it includes filename
// const body = new FormData();
// body.append("image", file);
// const resp = await fetch("/upload/image", {
// method: "POST",
// body,
// });
// if (resp.status === 200) {
// const data = await resp.json();
// // Add the file as an option and update the widget value
// if (!imageWidget.options.values.includes(data.name)) {
// imageWidget.options.values.push(data.name);
// }
// if (updateNode) {
// // showImage(data.name);
// imageWidget.value = data.name;
// }
// } else {
// alert(resp.status + " - " + resp.statusText);
// }
// } catch (error) {
// alert(error);
// }
// }
// const fileInput = document.createElement("input");
// Object.assign(fileInput, {
// type: "file",
// accept: "image/jpeg,image/png",
// style: "display: none",
// onchange: async () => {
// if (fileInput.files.length) {
// await uploadFile(fileInput.files[0], true);
// }
// },
// });
// document.body.append(fileInput);
// Create the button widget for selecting the files
uploadWidget = node.addWidget("button", "choose file to upload", "image", () => {
// fileInput.click();
});
uploadWidget.options = { serialize: false };
return { widget: uploadWidget };
const IMAGEUPLOAD: WidgetFactory = (node: LGraphNode, inputName: string, inputData: any, app): IComfyInputSlot => {
return addComfyInput(node, inputName, { type: "number", config: {} })
}
export type WidgetRepository = Record<string, WidgetFactory>
export const ComfyWidgets: WidgetRepository = {
"INT:seed": seedWidget,
"INT:noise_seed": seedWidget,
"INT:seed": INT,
"INT:noise_seed": INT,
FLOAT,
INT,
STRING,