Prepare for multiple workflows

This commit is contained in:
space-nuko
2023-05-20 17:09:00 -05:00
parent 3b9017dfad
commit a29afab9a7
12 changed files with 375 additions and 154 deletions

View File

@@ -1,4 +1,4 @@
import { LiteGraph, LGraph, LGraphCanvas, LGraphNode, type LGraphNodeConstructor, type LGraphNodeExecutable, type SerializedLGraph, type SerializedLGraphGroup, type SerializedLGraphNode, type SerializedLLink, NodeMode, type Vector2, BuiltInSlotType, type INodeInputSlot, type NodeID, type NodeTypeSpec, type NodeTypeOpts, type SlotIndex } from "@litegraph-ts/core";
import { LiteGraph, LGraph, LGraphCanvas, LGraphNode, type LGraphNodeConstructor, type LGraphNodeExecutable, type SerializedLGraph, type SerializedLGraphGroup, type SerializedLGraphNode, type SerializedLLink, NodeMode, type Vector2, BuiltInSlotType, type INodeInputSlot, type NodeID, type NodeTypeSpec, type NodeTypeOpts, type SlotIndex, type UUID } from "@litegraph-ts/core";
import type { LConnectionKind, INodeSlot } from "@litegraph-ts/core";
import ComfyAPI, { type ComfyAPIStatusResponse, type ComfyBoxPromptExtraData, type ComfyPromptRequest, type ComfyNodeID, type PromptID } from "$lib/api"
import { getPngMetadata, importA1111 } from "$lib/pnginfo";
@@ -21,7 +21,7 @@ import type ComfyGraphNode from "$lib/nodes/ComfyGraphNode";
import queueState from "$lib/stores/queueState";
import { type SvelteComponentDev } from "svelte/internal";
import type IComfyInputSlot from "$lib/IComfyInputSlot";
import type { SerializedLayoutState } from "$lib/stores/layoutState";
import type { LayoutState, SerializedLayoutState, WritableLayoutStateStore } from "$lib/stores/layoutState";
import layoutState from "$lib/stores/layoutState";
import { toast } from '@zerodevx/svelte-toast'
import ComfyGraph from "$lib/ComfyGraph";
@@ -33,7 +33,7 @@ import { download, graphToGraphVis, jsonToJsObject, promptToGraphVis, range, wor
import notify from "$lib/notify";
import configState from "$lib/stores/configState";
import { blankGraph } from "$lib/defaultGraph";
import type { ComfyExecutionResult } from "$lib/utils";
import type { SerializedPromptOutput } from "$lib/utils";
import ComfyPromptSerializer, { UpstreamNodeLocator, isActiveBackendNode } from "./ComfyPromptSerializer";
import { iterateNodeDefInputs, type ComfyNodeDef, isBackendNodeDefInputType, iterateNodeDefOutputs } from "$lib/ComfyNodeDef";
import { ComfyComboNode } from "$lib/nodes/widgets";
@@ -41,6 +41,7 @@ import parseA1111, { type A1111ParsedInfotext } from "$lib/parseA1111";
import convertA1111ToStdPrompt from "$lib/convertA1111ToStdPrompt";
import type { ComfyBoxStdPrompt } from "$lib/ComfyBoxStdPrompt";
import ComfyBoxStdPromptSerializer from "$lib/ComfyBoxStdPromptSerializer";
import { v4 as uuidv4 } from "uuid";
export const COMFYBOX_SERIAL_VERSION = 1;
@@ -51,12 +52,11 @@ if (typeof window !== "undefined") {
/*
* Queued prompt that hasn't been sent to the backend yet.
* TODO: Assumes the currently active graph will be serialized, needs to change
* for multiple loaded workflow support
*/
type QueueItem = {
type PromptQueueItem = {
num: number,
batchCount: number
workflow: ComfyWorkflow
}
export type A1111PromptAndInfo = {
@@ -111,7 +111,7 @@ export type SerializedPrompt = {
/*
* Outputs for each node.
*/
export type SerializedPromptOutputs = Record<ComfyNodeID, ComfyExecutionResult>
export type SerializedPromptOutputs = Record<ComfyNodeID, SerializedPromptOutput>
export type Progress = {
value: number,
@@ -134,20 +134,156 @@ type CanvasState = {
canvas: ComfyGraphCanvas,
}
type WorkflowState = {
title: string,
graph: ComfyGraph,
type ActiveCanvas = {
canvas: LGraphCanvas | null;
canvasHandler: () => void | null;
state: SerializedGraphCanvasState;
}
export type SerializedWorkflowState = {
graph: SerializedLGraph,
layout: SerializedLayoutState
}
/*
* ID for an opened workflow.
*
* Unlike NodeID and PromptID, these are *not* saved to the workflow itself.
* They are only used for identifying an open workflow in the program. If the
* workflow is closed and reopened, a different workflow ID will be assigned to
* it.
*/
export type WorkflowInstID = UUID;
export class ComfyWorkflow {
/*
* Used for uniquely identifying the instance of the opened workflow in the frontend.
*/
id: WorkflowInstID;
title: string;
graph: ComfyGraph;
layout: WritableLayoutStateStore;
canvases: Record<string, ActiveCanvas> = {};
constructor(title: string, graph: ComfyGraph, layout: WritableLayoutStateStore) {
this.id = uuidv4();
this.title = title;
this.layout = layout;
this.graph = graph;
}
start(key: string, canvas: ComfyGraphCanvas) {
if (this.canvases[key] != null)
throw new Error(`This workflow is already being displayed on canvas ${key}`)
const canvasHandler = () => canvas.draw(true);
this.canvases[key] = {
canvas,
canvasHandler,
state: {
// TODO
offset: [0, 0],
scale: 1
}
}
this.graph.attachCanvas(canvas);
this.graph.eventBus.on("afterExecute", canvasHandler)
if (Object.keys(this.canvases).length === 1)
this.graph.start();
}
stop(key: string) {
const canvas = this.canvases[key]
if (canvas == null)
throw new Error(`This workflow is not being displayed on canvas ${key}`)
this.graph.detachCanvas(canvas.canvas);
this.graph.eventBus.removeListener("afterExecute", canvas.canvasHandler)
delete this.canvases[key]
if (Object.keys(this.canvases).length === 0)
this.graph.stop();
}
stopAll() {
for (const key of Object.keys(this.canvases))
this.stop(key)
this.graph.stop()
}
serialize(): SerializedWorkflowState {
const graph = this.graph;
const serializedGraph = graph.serialize()
const serializedLayout = this.layout.serialize()
return {
graph: serializedGraph,
layout: serializedLayout
}
}
static deserialize(data: SerializedWorkflowState): ComfyWorkflow {
const layout = layoutState; // TODO
// Ensure loadGraphData does not trigger any state changes in layoutState
// (isConfiguring is set to true here)
// lGraph.configure will add new nodes, triggering onNodeAdded, but we
// want to restore the layoutState ourselves
layout.onStartConfigure();
// Patch T2IAdapterLoader to ControlNetLoader since they are the same node now
for (let n of data.graph.nodes) {
if (n.type == "T2IAdapterLoader") n.type = "ControlNetLoader";
}
const graph = new ComfyGraph();
graph.configure(data.graph);
for (const node of graph._nodes) {
const size = node.computeSize();
size[0] = Math.max(node.size[0], size[0]);
size[1] = Math.max(node.size[1], size[1]);
node.size = size;
// this.#invokeExtensions("loadedGraphNode", node);
}
// Now restore the layout
// Subsequent added nodes will add the UI data to layoutState
// TODO
layout.deserialize(data.layout, graph)
return new ComfyWorkflow("Workflow X", graph, layout);
}
}
export default class ComfyApp {
api: ComfyAPI;
rootEl: HTMLDivElement | null = null;
canvasEl: HTMLCanvasElement | null = null;
canvasCtx: CanvasRenderingContext2D | null = null;
lGraph: ComfyGraph | null = null;
lCanvas: ComfyGraphCanvas | null = null;
dropZone: HTMLElement | null = null;
nodeOutputs: Record<string, any> = {};
openedWorkflows: ComfyWorkflow[] = [];
openedWorkflowsByID: Record<WorkflowInstID, ComfyWorkflow> = {};
activeWorkflowIdx: number = -1;
get activeWorkflow(): ComfyWorkflow | null {
return this.openedWorkflows[this.activeWorkflowIdx]
}
get activeGraph(): ComfyGraph | null {
return this.activeWorkflow?.graph;
}
getWorkflow(id: WorkflowInstID): ComfyWorkflow | null {
return this.openedWorkflowsByID[id];
}
shiftDown: boolean = false;
ctrlDown: boolean = false;
@@ -155,7 +291,7 @@ export default class ComfyApp {
alreadySetup: Writable<boolean> = writable(false);
a1111Prompt: Writable<A1111PromptAndInfo | null> = writable(null);
private queueItems: QueueItem[] = [];
private queueItems: PromptQueueItem[] = [];
private processingQueue: boolean = false;
private promptSerializer: ComfyPromptSerializer;
private stdPromptSerializer: ComfyBoxStdPromptSerializer;
@@ -176,8 +312,7 @@ export default class ComfyApp {
this.rootEl = document.getElementById("app-root") as HTMLDivElement;
this.canvasEl = document.getElementById("graph-canvas") as HTMLCanvasElement;
this.lGraph = new ComfyGraph();
this.lCanvas = new ComfyGraphCanvas(this, this.canvasEl);
this.lCanvas = new ComfyGraphCanvas(this, null, this.canvasEl);
this.canvasCtx = this.canvasEl.getContext("2d");
const uiUnlocked = get(uiState).uiUnlocked;
@@ -193,7 +328,7 @@ export default class ComfyApp {
const json = localStorage.getItem("workflow");
if (json) {
const state = JSON.parse(json) as SerializedAppState;
await this.deserialize(state)
await this.openWorkflow(state)
restored = true;
}
} catch (err) {
@@ -236,6 +371,24 @@ export default class ComfyApp {
this.lCanvas.draw(true, true);
}
serialize(): SerializedAppState {
const workflow = this.activeWorkflow
if (workflow == null)
throw new Error("No workflow active!")
const { graph, layout } = workflow.serialize();
const canvas = this.lCanvas.serialize();
return {
createdBy: "ComfyBox",
version: COMFYBOX_SERIAL_VERSION,
commitHash: __GIT_COMMIT_HASH__,
workflow: graph,
layout,
canvas
}
}
saveStateToLocalStorage() {
try {
uiState.update(s => { s.isSavingToLocalStorage = true; return s; })
@@ -362,8 +515,12 @@ export default class ComfyApp {
} catch (error) { }
}
if (workflow && workflow.version && workflow.nodes && workflow.extra) {
this.loadGraphData(workflow);
if (workflow && workflow.createdBy === "ComfyBox") {
this.openWorkflow(workflow);
}
else {
// TODO handle vanilla workflows
throw new Error("Workflow was not in ComfyBox format!")
}
});
}
@@ -386,21 +543,27 @@ export default class ComfyApp {
this.api.addEventListener("progress", (progress: Progress) => {
queueState.progressUpdated(progress);
this.lGraph.setDirtyCanvas(true, false);
this.activeGraph?.setDirtyCanvas(true, false); // TODO PromptID
});
this.api.addEventListener("executing", (promptID: PromptID | null, nodeID: ComfyNodeID | null) => {
queueState.executingUpdated(promptID, nodeID);
this.lGraph.setDirtyCanvas(true, false);
const queueEntry = queueState.executingUpdated(promptID, nodeID);
if (queueEntry != null) {
const workflow = this.getWorkflow(queueEntry.workflowID)
workflow?.graph.setDirtyCanvas(true, false);
}
});
this.api.addEventListener("executed", (promptID: PromptID, nodeID: ComfyNodeID, output: ComfyExecutionResult) => {
this.nodeOutputs[nodeID] = output;
const node = this.lGraph.getNodeByIdRecursive(nodeID) as ComfyGraphNode;
if (node?.onExecuted) {
node.onExecuted(output);
this.api.addEventListener("executed", (promptID: PromptID, nodeID: ComfyNodeID, output: SerializedPromptOutput) => {
const queueEntry = queueState.onExecuted(promptID, nodeID, output)
if (queueEntry != null) {
const workflow = this.getWorkflow(queueEntry.workflowID)
workflow?.graph.setDirtyCanvas(true, false);
const node = workflow?.graph.getNodeByIdRecursive(nodeID) as ComfyGraphNode;
if (node?.onExecuted) {
node.onExecuted(output);
}
}
queueState.onExecuted(promptID, nodeID, output)
});
this.api.addEventListener("execution_start", (promptID: PromptID) => {
@@ -467,46 +630,68 @@ export default class ComfyApp {
setColor(BuiltInSlotType.ACTION, "lightseagreen")
}
serialize(): SerializedAppState {
const graph = this.lGraph;
const serializedGraph = graph.serialize()
const serializedLayout = layoutState.serialize()
const serializedCanvas = this.lCanvas.serialize();
return {
createdBy: "ComfyBox",
version: COMFYBOX_SERIAL_VERSION,
workflow: serializedGraph,
layout: serializedLayout,
canvas: serializedCanvas
}
createNewWorkflow(): ComfyWorkflow {
// TODO remove
const workflow = ComfyWorkflow.deserialize({ graph: blankGraph.workflow, layout: blankGraph.layout })
this.openedWorkflows.push(workflow);
this.setActiveWorkflow(this.openedWorkflows.length - 1)
return workflow;
}
async deserialize(data: SerializedAppState) {
async openWorkflow(data: SerializedAppState): Promise<ComfyWorkflow> {
if (data.version !== COMFYBOX_SERIAL_VERSION) {
throw `Invalid ComfyBox saved data format: ${data.version}`
}
// Ensure loadGraphData does not trigger any state changes in layoutState
// (isConfiguring is set to true here)
// lGraph.configure will add new nodes, triggering onNodeAdded, but we
// want to restore the layoutState ourselves
layoutState.onStartConfigure();
this.clean();
this.loadGraphData(data.workflow)
// Now restore the layout
// Subsequent added nodes will add the UI data to layoutState
layoutState.deserialize(data.layout, this.lGraph)
const workflow = ComfyWorkflow.deserialize({ graph: data.workflow, layout: data.layout })
// Restore canvas offset/zoom
this.lCanvas.deserialize(data.canvas)
await this.refreshComboInNodes();
await this.refreshComboInNodes(workflow);
this.lGraph.start();
this.lGraph.eventBus.on("afterExecute", () => this.lCanvas.draw(true))
this.openedWorkflows.push(workflow);
this.setActiveWorkflow(this.openedWorkflows.length - 1)
return workflow;
}
closeWorkflow(index: number) {
if (index < 0 || index >= this.openedWorkflows.length)
return;
const workflow = this.openedWorkflows[index];
workflow.stopAll();
this.openedWorkflows.splice(index, 1)
this.setActiveWorkflow(0);
}
closeAllWorkflows() {
while (this.openedWorkflows.length > 0)
this.closeWorkflow(0)
}
setActiveWorkflow(index: number) {
if (this.openedWorkflows.length === 0) {
this.activeWorkflowIdx = -1;
return;
}
if (index < 0 || index >= this.openedWorkflows.length || this.activeWorkflowIdx === index)
return;
if (this.activeWorkflow != null)
this.activeWorkflow.stop("app")
const workflow = this.openedWorkflows[index]
this.activeWorkflowIdx = index;
console.warn("START")
workflow.start("app", this.lCanvas);
this.lCanvas.deserialize(workflow.canvases["app"].state)
}
async initDefaultGraph() {
@@ -520,50 +705,14 @@ export default class ComfyApp {
notify(`Failed to load default graph: ${error}`, { type: "error" })
state = structuredClone(blankGraph)
}
await this.deserialize(state)
}
/**
* Populates the graph with the specified workflow data
* @param {*} graphData A serialized graph object
*/
loadGraphData(graphData: SerializedLGraph) {
this.clean();
// Patch T2IAdapterLoader to ControlNetLoader since they are the same node now
for (let n of graphData.nodes) {
if (n.type == "T2IAdapterLoader") n.type = "ControlNetLoader";
}
this.lGraph.configure(graphData);
for (const node of this.lGraph._nodes) {
const size = node.computeSize();
size[0] = Math.max(node.size[0], size[0]);
size[1] = Math.max(node.size[1], size[1]);
node.size = size;
// this.#invokeExtensions("loadedGraphNode", node);
}
await this.openWorkflow(state)
}
clear() {
this.clean();
const blankGraph: SerializedLGraph = {
last_node_id: 0,
last_link_id: 0,
nodes: [],
links: [],
groups: [],
config: {},
extra: {},
version: 0
}
layoutState.onStartConfigure();
this.lCanvas.closeAllSubgraphs();
this.lGraph.configure(blankGraph)
layoutState.initDefaultLayout();
this.closeAllWorkflows();
uiState.update(s => {
s.uiUnlocked = true;
s.uiEditMode = "widgets";
@@ -572,7 +721,10 @@ export default class ComfyApp {
}
runDefaultQueueAction() {
for (const node of this.lGraph.iterateNodesInOrderRecursive()) {
if (this.activeWorkflow == null)
return;
for (const node of this.activeGraph.iterateNodesInOrderRecursive()) {
if ("onDefaultQueueAction" in node) {
(node as ComfyGraphNode).onDefaultQueueAction()
}
@@ -614,12 +766,17 @@ export default class ComfyApp {
* Converts the current graph workflow for sending to the API
* @returns The workflow and node links
*/
graphToPrompt(tag: string | null = null): SerializedPrompt {
return this.promptSerializer.serialize(this.lGraph, tag)
graphToPrompt(workflow: ComfyWorkflow, tag: string | null = null): SerializedPrompt {
return this.promptSerializer.serialize(workflow.graph, tag)
}
async queuePrompt(num: number, batchCount: number = 1, tag: string | null = null) {
this.queueItems.push({ num, batchCount });
if (this.activeWorkflow === null) {
notify("No workflow is opened!", { type: "error" })
return;
}
this.queueItems.push({ num, batchCount, workflow: this.activeWorkflow });
// Only have one action process the items so each one gets a unique seed correctly
if (this.processingQueue) {
@@ -630,13 +787,15 @@ export default class ComfyApp {
tag = null;
this.processingQueue = true;
let workflow;
try {
while (this.queueItems.length) {
({ num, batchCount } = this.queueItems.pop());
({ num, batchCount, workflow } = this.queueItems.pop());
console.debug(`Queue get! ${num} ${batchCount} ${tag}`);
const thumbnails = []
for (const node of this.lGraph.iterateNodesInOrderRecursive()) {
for (const node of workflow.graph.iterateNodesInOrderRecursive()) {
if (node.mode !== NodeMode.ALWAYS
|| (tag != null
&& Array.isArray(node.properties.tags)
@@ -651,7 +810,7 @@ export default class ComfyApp {
}
for (let i = 0; i < batchCount; i++) {
for (const node of this.lGraph.iterateNodesInOrderRecursive()) {
for (const node of workflow.graph.iterateNodesInOrderRecursive()) {
if (node.mode !== NodeMode.ALWAYS)
continue;
@@ -660,9 +819,9 @@ export default class ComfyApp {
}
}
const p = this.graphToPrompt(tag);
const p = this.graphToPrompt(workflow, tag);
const l = layoutState.serialize();
console.debug(graphToGraphVis(this.lGraph))
console.debug(graphToGraphVis(workflow.graph))
console.debug(promptToGraphVis(p))
const stdPrompt = this.stdPromptSerializer.serialize(p);
@@ -692,7 +851,7 @@ export default class ComfyApp {
error = response.error;
}
else {
queueState.afterQueued(response.promptID, num, p.output, extraData)
queueState.afterQueued(workflow.id, response.promptID, num, p.output, extraData)
}
} catch (err) {
error = err?.toString();
@@ -701,13 +860,13 @@ export default class ComfyApp {
if (error != null) {
const mes: string = error;
notify(`Error queuing prompt:\n${mes}`, { type: "error" })
console.error(graphToGraphVis(this.lGraph))
console.error(graphToGraphVis(workflow.graph))
console.error(promptToGraphVis(p))
console.error("Error queuing prompt", error, num, p)
break;
}
for (const node of this.lGraph.iterateNodesInOrderRecursive()) {
for (const node of workflow.graph.iterateNodesInOrderRecursive()) {
if ("afterQueued" in node) {
(node as ComfyGraphNode).afterQueued(p, tag);
}
@@ -730,7 +889,7 @@ export default class ComfyApp {
const pngInfo = await getPngMetadata(file);
if (pngInfo) {
if (pngInfo.comfyBoxConfig) {
this.deserialize(JSON.parse(pngInfo.comfyBoxConfig));
await this.openWorkflow(JSON.parse(pngInfo.comfyBoxConfig));
} else if (pngInfo.parameters) {
const parsed = parseA1111(pngInfo.parameters)
if ("error" in parsed) {
@@ -752,8 +911,8 @@ export default class ComfyApp {
}
} else if (file.type === "application/json" || file.name.endsWith(".json")) {
const reader = new FileReader();
reader.onload = () => {
this.deserialize(JSON.parse(reader.result as string));
reader.onload = async () => {
await this.openWorkflow(JSON.parse(reader.result as string));
};
reader.readAsText(file);
}
@@ -772,7 +931,7 @@ export default class ComfyApp {
/**
* Refresh combo list on whole nodes
*/
async refreshComboInNodes(flashUI: boolean = false) {
async refreshComboInNodes(workflow: ComfyWorkflow, flashUI: boolean = false) {
const defs = await this.api.getNodeDefs();
const isComfyComboNode = (node: LGraphNode): node is ComfyComboNode => {
@@ -816,7 +975,7 @@ export default class ComfyApp {
return result
}
for (const node of this.lGraph.iterateNodesInOrderRecursive()) {
for (const node of workflow.graph.iterateNodesInOrderRecursive()) {
if (!isActiveBackendNode(node))
continue;
@@ -838,7 +997,7 @@ export default class ComfyApp {
console.debug("[refreshComboInNodes] found:", backendUpdatedCombos.length, backendUpdatedCombos)
// Mark combo nodes without backend configs as being loaded already.
for (const node of this.lGraph.iterateNodesOfClassRecursive(ComfyComboNode)) {
for (const node of workflow.graph.iterateNodesOfClassRecursive(ComfyComboNode)) {
if (backendUpdatedCombos[node.id] != null) {
continue;
}
@@ -870,12 +1029,15 @@ export default class ComfyApp {
// Load definitions from the backend.
for (const { comboNode, comfyInput, backendNode } of Object.values(backendUpdatedCombos)) {
const def = defs[backendNode.type];
const rawValues = def["input"]["required"][comfyInput.name][0];
const [rawValues, opts] = def.input.required[comfyInput.name];
console.debug("[ComfyApp] Reconfiguring combo widget", backendNode.type, "=>", comboNode.type, rawValues.length)
comboNode.doAutoConfig(comfyInput, { includeProperties: new Set(["values"]), setWidgetTitle: false })
comboNode.formatValues(rawValues as string[], true)
const values = rawValues as string[]
const defaultValue = rawValues[0];
comboNode.formatValues(values, defaultValue, true)
if (!rawValues?.includes(get(comboNode.value))) {
comboNode.setValue(rawValues[0], comfyInput.config.defaultValue)
}
@@ -886,7 +1048,6 @@ export default class ComfyApp {
* Clean current state
*/
clean() {
this.nodeOutputs = {};
this.a1111Prompt.set(null);
}
}

View File

@@ -275,6 +275,12 @@
Error loading app
</div>
<div>{error}</div>
{#if error.stack}
{@const lines = error.stack.split("\n")}
{#each lines as line}
<div style:font-size="16px">{line}</div>
{/each}
{/if}
</div>
{/await}
{/if}