Conversion to standard prompt format

src/lib/ComfyBoxStdPrompt.ts

@@ -7,11 +7,10 @@ const ModelHashes = z.object({
     a1111_shorthash !== undefined || sha256 !== undefined,
     { message: "At least one model hash must be specified" })
 
-const GroupPrompt = z.object({
-    positive: z.string(),
-    negative: z.string()
+const GroupConditioning = z.object({
+    text: z.string(),
 })
-export type ComfyBoxStdGroupPrompt = z.infer<typeof GroupPrompt>
+export type ComfyBoxStdGroupConditioning = z.infer<typeof GroupConditioning>
 
 const GroupCheckpoint = z.object({
     model_name: z.string().optional(),
@@ -138,15 +137,46 @@ const GroupDDetailer = z.object({
     dilation: z.number(),
     offset_x: z.number(),
     offset_y: z.number(),
-    inpaint_full: z.number(),
+    preprocess: z.boolean(),
+    inpaint_full: z.boolean(),
     inpaint_padding: z.number(),
+    cfg: z.number()
 })
 export type ComfyBoxStdGroupDDetailer = z.infer<typeof GroupDDetailer>
 
-const group = (s: ZodTypeAny) => z.optional(z.array(s).nonempty());
+/*
+ * This metadata can be attached to each entry in a group to assist in
+ * identifying the correct nodes to apply it to.
+ *
+ * As an example, positive and negative conditioning are deployed as two
+ * separate nodes in ComfyUI. This makes bundling them into a { positive,
+ * negative } entry difficult as either one can be missing. So instead they're
+ * tagged like
+ *
+ * {
+ *   conditioning: [
+ *     { text: "masterpiece",   "$meta": { types: ["positive"] } },
+ *     { text: "worst quality", "$meta": { types: ["negative"] } },
+ *   ]
+ * }
+ *
+ * The reasoning is the "types" information isn't required to reinstantiate
+ * the node, it's only semantic information describing how the node is used in
+ * the encompassing workflow. When the prompt is loaded the workflow can be
+ * searched for a node with the compatible type to attach the information to.
+ */
+const GroupMetadata = z.object({
+    types: z.array(z.string()).nonempty().optional()
+})
+export type ComfyBoxStdGroupMetadata = z.infer<typeof GroupMetadata>
+
+const group = (entry: ZodTypeAny) => {
+    const groupEntry = entry.and(z.object({ "$meta": GroupMetadata }))
+    return z.optional(z.array(groupEntry).nonempty());
+}
 
 const Parameters = z.object({
-    prompt: group(GroupPrompt),
+    conditioning: group(GroupConditioning),
     checkpoint: group(GroupCheckpoint),
     vae: group(GroupVAE),
     k_sampler: group(GroupKSampler),
@@ -183,22 +213,40 @@ const Metadata = z.object({
     extra_data: ExtraData
 })
 
-const Prompt = z.object({
-    metadata: Metadata,
-    parameters: Parameters
-})
-
 const ComfyBoxStdPrompt = z.object({
     version: z.number(),
-    prompt: Prompt,
+    metadata: Metadata,
+    parameters: Parameters
 })
 
 export default ComfyBoxStdPrompt
 
 /*
- * A standardized Stable Diffusion prompt and parameter format, to be used with
- * an encompassing workflow. Aims to encompass an arbitrary number of parameter
+ * A standardized Stable Diffusion parameter format that should be used with an
+ * encompassing workflow. Aims to encompass an arbitrary number of parameter
  * counts and types, so that most ComfyUI workflows can have parts of their
  * prompts transferred between each other.
+ *
+ * This format does *not* describe how the information should be used in the
+ * underlying workflow, i.e. it does not specify the structure of a ComfyUI
+ * execution graph. It only gives hints via tagged input types on each input
+ * entry as to where the data should be inserted. To recreate a ComfyBox
+ * workflow with the exact state of the UI intact, the `SerializedAppState` type
+ * should be used instead. It suffices to embed data of that type in the output
+ * PNGs for recreating their workflows. This type is meant as an interchange
+ * format *between* workflows so their inputs can be copied to and from each
+ * other in a sane-enough manner. (In ComfyBox, copying workflow outputs like
+ * images to other workflows is handled separately, since this type does not
+ * retain the actual image data.)
+ *
+ * In contrast with a serialized workflow, which is concerned with the
+ * connections between nodes and the state of the frontend's UI, this format
+ * concerns itself with the exact values that the execution backend receives,
+ * after the data in the UI have finished processing.
+ *
+ * (Take for example a "scale by" slider that adjusts the width and height of an
+ * img2img input image of 512 x 512 resolution by 2x. The backend will only
+ * "see" width 1024 and height 1024, even though the only parameter exposed from
+ * the frontend was the scale of 2.)
  */
 export type ComfyBoxStdPrompt = z.infer<typeof ComfyBoxStdPrompt>

src/lib/api.ts

@@ -1,4 +1,4 @@
-import type { Progress, SerializedPrompt, SerializedPromptInputs, SerializedPromptInputsAll, SerializedPromptOutputs } from "./components/ComfyApp";
+import type { Progress, SerializedPrompt, SerializedPromptInputsForNode, SerializedPromptInputsAll, SerializedPromptOutputs } from "./components/ComfyApp";
 import type TypedEmitter from "typed-emitter";
 import EventEmitter from "events";
 import type { ComfyImageLocation } from "$lib/utils";

src/lib/comfyStdPromptConverters.ts

@@ -0,0 +1,26 @@
+import type { ComfyBoxStdGroupLoRA, ComfyBoxStdPrompt } from "./ComfyBoxStdPrompt";
+import type { ComfyNodeID } from "./api";
+import type { SerializedPromptInputs } from "./components/ComfyApp";
+import type { ComfyBackendNode } from "./nodes/ComfyBackendNode";
+
+export type ComfyPromptConverter = (stdPrompt: ComfyBoxStdPrompt, inputs: SerializedPromptInputs, nodeID: ComfyNodeID) => void;
+
+function LoraLoader(stdPrompt: ComfyBoxStdPrompt, inputs: SerializedPromptInputs) {
+    const params = stdPrompt.prompt.parameters
+
+    const lora: ComfyBoxStdGroupLoRA = {
+        model_name: inputs["lora_name"],
+        strength_unet: inputs["strength_model"],
+        strength_tenc: inputs["strength_clip"]
+    }
+
+    if (params.lora)
+        params.lora.push(lora)
+    else
+        params.lora = [lora]
+}
+
+const converters: Record<string, ComfyPromptConverter> = {
+    LoraLoader
+}
+export default converters;

src/lib/components/ComfyApp.ts

@@ -40,6 +40,7 @@ import { ComfyComboNode } from "$lib/nodes/widgets";
 import parseA1111, { type A1111ParsedInfotext } from "$lib/parseA1111";
 import convertA1111ToStdPrompt from "$lib/convertA1111ToStdPrompt";
 import type { ComfyBoxStdPrompt } from "$lib/ComfyBoxStdPrompt";
+import ComfyBoxStdPromptSerializer from "./ComfyBoxStdPromptSerializer";
 
 export const COMFYBOX_SERIAL_VERSION = 1;
 
@@ -86,19 +87,21 @@ export type SerializedAppState = {
 /** [link_origin, link_slot_index] | input_value */
 export type SerializedPromptInput = [ComfyNodeID, number] | any
 
+export type SerializedPromptInputs = Record<string, SerializedPromptInput>;
+
 /*
  * A single node in the prompt and its input values.
  */
-export type SerializedPromptInputs = {
+export type SerializedPromptInputsForNode = {
     /* property name -> value or link */
-    inputs: Record<string, SerializedPromptInput>,
+    inputs: SerializedPromptInputs,
     class_type: string
 }
 
 /*
  * All nodes in the graph and their input values.
  */
-export type SerializedPromptInputsAll = Record<ComfyNodeID, SerializedPromptInputs>
+export type SerializedPromptInputsAll = Record<ComfyNodeID, SerializedPromptInputsForNode>
 
 export type SerializedPrompt = {
     workflow: SerializedLGraph,
@@ -144,10 +147,12 @@ export default class ComfyApp {
     private queueItems: QueueItem[] = [];
     private processingQueue: boolean = false;
     private promptSerializer: ComfyPromptSerializer;
+    private stdPromptSerializer: ComfyBoxStdPromptSerializer;
 
     constructor() {
         this.api = new ComfyAPI();
         this.promptSerializer = new ComfyPromptSerializer();
+        this.stdPromptSerializer = new ComfyBoxStdPromptSerializer();
     }
 
     async setup(): Promise<void> {
@@ -649,6 +654,9 @@ export default class ComfyApp {
                     console.debug(graphToGraphVis(this.lGraph))
                     console.debug(promptToGraphVis(p))
 
+                    const stdPrompt = this.stdPromptSerializer.serialize(p);
+                    console.warn("STD", stdPrompt);
+
                     const extraData: ComfyBoxPromptExtraData = {
                         extra_pnginfo: {
                             workflow: p.workflow,

src/lib/components/ComfyBoxStdPromptSerializer.ts

@@ -0,0 +1,35 @@
+import type { ComfyBoxStdPrompt } from "$lib/ComfyBoxStdPrompt";
+import type { SerializedPrompt } from "./ComfyApp";
+import comfyStdPromptConverters from "$lib/comfyStdPromptConverters"
+
+const COMMIT_HASH: string = "asdf";
+
+export default class ComfyBoxStdPromptSerializer {
+    serialize(prompt: SerializedPrompt): ComfyBoxStdPrompt {
+        const stdPrompt: ComfyBoxStdPrompt = {
+            version: 1,
+            metadata: {
+                created_with: "ComfyBox",
+                commit_hash: COMMIT_HASH,
+                extra_data: {
+                    comfybox: {
+                    }
+                }
+            },
+            parameters: {}
+        }
+
+        for (const [nodeID, inputs] of Object.entries(prompt.output)) {
+            const classType = inputs.class_type
+            const converter = comfyStdPromptConverters[classType]
+            if (converter) {
+                converter(stdPrompt, inputs.inputs, nodeID)
+            }
+            else {
+                console.warn("No StdPrompt type converter for comfy class!", classType)
+            }
+        }
+
+        return stdPrompt
+    }
+}

src/lib/components/ComfyPromptSerializer.ts

@@ -2,7 +2,7 @@ import type ComfyGraph from "$lib/ComfyGraph";
 import type { ComfyBackendNode } from "$lib/nodes/ComfyBackendNode";
 import type ComfyGraphNode from "$lib/nodes/ComfyGraphNode";
 import { GraphInput, GraphOutput, LGraph, LGraphNode, LLink, NodeMode, Subgraph, type SlotIndex } from "@litegraph-ts/core";
-import type { SerializedPrompt, SerializedPromptInput, SerializedPromptInputs, SerializedPromptInputsAll } from "./ComfyApp";
+import type { SerializedPrompt, SerializedPromptInput, SerializedPromptInputsForNode, SerializedPromptInputsAll, SerializedPromptInputs } from "./ComfyApp";
 import type IComfyInputSlot from "$lib/IComfyInputSlot";
 
 function hasTag(node: LGraphNode, tag: string): boolean {
@@ -150,7 +150,7 @@ export class UpstreamNodeLocator {
 }
 
 export default class ComfyPromptSerializer {
-    serializeInputValues(node: ComfyBackendNode): Record<string, SerializedPromptInput> {
+    serializeInputValues(node: ComfyBackendNode): SerializedPromptInputs {
         // Store input values passed by frontend-only nodes
         if (!node.inputs) {
             return {}

src/lib/convertA1111ToStdPrompt.ts

@@ -49,10 +49,20 @@ export default function convertA1111ToStdPrompt(infotext: A1111ParsedInfotext):
 
     const parameters: ComfyBoxStdParameters = {}
 
-    parameters.prompt = [{
-        positive: infotext.positive,
-        negative: infotext.negative,
-    }]
+    parameters.conditioning = [
+        {
+            "^meta": {
+                types: ["positive"]
+            },
+            text: infotext.positive,
+        },
+        {
+            "^meta": {
+                types: ["negative"]
+            },
+            text: infotext.negative,
+        }
+    ]
 
     const hrUp = popOpt("hires upscale");
     const hrSz = popOpt("hires resize");
@@ -327,18 +337,16 @@ export default function convertA1111ToStdPrompt(infotext: A1111ParsedInfotext):
 
     const prompt: ComfyBoxStdPrompt = {
         version: 1,
-        prompt: {
-            metadata: {
-                created_with: "stable-diffusion-webui",
-                app_version,
-                extra_data: {
-                    a1111: {
-                        params: infotext.extraParams
-                    }
+        metadata: {
+            created_with: "stable-diffusion-webui",
+            app_version,
+            extra_data: {
+                a1111: {
+                    params: infotext.extraParams
                 }
-            },
-            parameters
-        }
+            }
+        },
+        parameters
     }
 
     console.warn("Unhandled A1111 parameters:", infotext.extraParams, infotext.extraNetworks)