neuro_llc/ComfyBox
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 1 Wiki Activity

Conversion to standard prompt format

Browse Source
This commit is contained in:
space-nuko
2023-05-19 11:05:35 -05:00
parent 74a1b5c636
commit ec80884684
10 changed files with 404 additions and 224 deletions
Download Patch File Download Diff File Expand all files Collapse all files

78
src/lib/ComfyBoxStdPrompt.ts
View File

@@ -7,11 +7,10 @@ const ModelHashes = z.object({
a1111_shorthash !== undefined || sha256 !== undefined,
{ message: "At least one model hash must be specified" })
const GroupPrompt = z.object({
positive: z.string(),
negative: z.string()
const GroupConditioning = z.object({
text: z.string(),
})
export type ComfyBoxStdGroupPrompt = z.infer<typeof GroupPrompt>
export type ComfyBoxStdGroupConditioning = z.infer<typeof GroupConditioning>
const GroupCheckpoint = z.object({
model_name: z.string().optional(),
@@ -138,15 +137,46 @@ const GroupDDetailer = z.object({
dilation: z.number(),
offset_x: z.number(),
offset_y: z.number(),
inpaint_full: z.number(),
preprocess: z.boolean(),
inpaint_full: z.boolean(),
inpaint_padding: z.number(),
cfg: z.number()
})
export type ComfyBoxStdGroupDDetailer = z.infer<typeof GroupDDetailer>
const group = (s: ZodTypeAny) => z.optional(z.array(s).nonempty());
/*
* This metadata can be attached to each entry in a group to assist in
* identifying the correct nodes to apply it to.
*
* As an example, positive and negative conditioning are deployed as two
* separate nodes in ComfyUI. This makes bundling them into a { positive,
* negative } entry difficult as either one can be missing. So instead they're
* tagged like
*
* {
* conditioning: [
* { text: "masterpiece", "$meta": { types: ["positive"] } },
* { text: "worst quality", "$meta": { types: ["negative"] } },
* ]
* }
*
* The reasoning is the "types" information isn't required to reinstantiate
* the node, it's only semantic information describing how the node is used in
* the encompassing workflow. When the prompt is loaded the workflow can be
* searched for a node with the compatible type to attach the information to.
*/
const GroupMetadata = z.object({
types: z.array(z.string()).nonempty().optional()
})
export type ComfyBoxStdGroupMetadata = z.infer<typeof GroupMetadata>
const group = (entry: ZodTypeAny) => {
const groupEntry = entry.and(z.object({ "$meta": GroupMetadata }))
return z.optional(z.array(groupEntry).nonempty());
}
const Parameters = z.object({
prompt: group(GroupPrompt),
conditioning: group(GroupConditioning),
checkpoint: group(GroupCheckpoint),
vae: group(GroupVAE),
k_sampler: group(GroupKSampler),
@@ -183,22 +213,40 @@ const Metadata = z.object({
extra_data: ExtraData
})
const Prompt = z.object({
metadata: Metadata,
parameters: Parameters
})
const ComfyBoxStdPrompt = z.object({
version: z.number(),
prompt: Prompt,
metadata: Metadata,
parameters: Parameters
})
export default ComfyBoxStdPrompt
/*
* A standardized Stable Diffusion prompt and parameter format, to be used with
* an encompassing workflow. Aims to encompass an arbitrary number of parameter
* A standardized Stable Diffusion parameter format that should be used with an
* encompassing workflow. Aims to encompass an arbitrary number of parameter
* counts and types, so that most ComfyUI workflows can have parts of their
* prompts transferred between each other.
*
* This format does *not* describe how the information should be used in the
* underlying workflow, i.e. it does not specify the structure of a ComfyUI
* execution graph. It only gives hints via tagged input types on each input
* entry as to where the data should be inserted. To recreate a ComfyBox
* workflow with the exact state of the UI intact, the `SerializedAppState` type
* should be used instead. It suffices to embed data of that type in the output
* PNGs for recreating their workflows. This type is meant as an interchange
* format *between* workflows so their inputs can be copied to and from each
* other in a sane-enough manner. (In ComfyBox, copying workflow outputs like
* images to other workflows is handled separately, since this type does not
* retain the actual image data.)
*
* In contrast with a serialized workflow, which is concerned with the
* connections between nodes and the state of the frontend's UI, this format
* concerns itself with the exact values that the execution backend receives,
* after the data in the UI have finished processing.
*
* (Take for example a "scale by" slider that adjusts the width and height of an
* img2img input image of 512 x 512 resolution by 2x. The backend will only
* "see" width 1024 and height 1024, even though the only parameter exposed from
* the frontend was the scale of 2.)
*/
export type ComfyBoxStdPrompt = z.infer<typeof ComfyBoxStdPrompt>

2
src/lib/api.ts
View File

@@ -1,4 +1,4 @@
import type { Progress, SerializedPrompt, SerializedPromptInputs, SerializedPromptInputsAll, SerializedPromptOutputs } from "./components/ComfyApp";
import type { Progress, SerializedPrompt, SerializedPromptInputsForNode, SerializedPromptInputsAll, SerializedPromptOutputs } from "./components/ComfyApp";
import type TypedEmitter from "typed-emitter";
import EventEmitter from "events";
import type { ComfyImageLocation } from "$lib/utils";

26
src/lib/comfyStdPromptConverters.ts Normal file
View File

@@ -0,0 +1,26 @@
import type { ComfyBoxStdGroupLoRA, ComfyBoxStdPrompt } from "./ComfyBoxStdPrompt";
import type { ComfyNodeID } from "./api";
import type { SerializedPromptInputs } from "./components/ComfyApp";
import type { ComfyBackendNode } from "./nodes/ComfyBackendNode";
export type ComfyPromptConverter = (stdPrompt: ComfyBoxStdPrompt, inputs: SerializedPromptInputs, nodeID: ComfyNodeID) => void;
function LoraLoader(stdPrompt: ComfyBoxStdPrompt, inputs: SerializedPromptInputs) {
const params = stdPrompt.prompt.parameters
const lora: ComfyBoxStdGroupLoRA = {
model_name: inputs["lora_name"],
strength_unet: inputs["strength_model"],
strength_tenc: inputs["strength_clip"]
}
if (params.lora)
params.lora.push(lora)
else
params.lora = [lora]
}
const converters: Record<string, ComfyPromptConverter> = {
LoraLoader
}
export default converters;

14
src/lib/components/ComfyApp.ts
View File

@@ -40,6 +40,7 @@ import { ComfyComboNode } from "$lib/nodes/widgets";
import parseA1111, { type A1111ParsedInfotext } from "$lib/parseA1111";
import convertA1111ToStdPrompt from "$lib/convertA1111ToStdPrompt";
import type { ComfyBoxStdPrompt } from "$lib/ComfyBoxStdPrompt";
import ComfyBoxStdPromptSerializer from "./ComfyBoxStdPromptSerializer";
export const COMFYBOX_SERIAL_VERSION = 1;
@@ -86,19 +87,21 @@ export type SerializedAppState = {
/** [link_origin, link_slot_index] | input_value */
export type SerializedPromptInput = [ComfyNodeID, number] | any
export type SerializedPromptInputs = Record<string, SerializedPromptInput>;
/*
* A single node in the prompt and its input values.
*/
export type SerializedPromptInputs = {
export type SerializedPromptInputsForNode = {
/* property name -> value or link */
inputs: Record<string, SerializedPromptInput>,
inputs: SerializedPromptInputs,
class_type: string
}
/*
* All nodes in the graph and their input values.
*/
export type SerializedPromptInputsAll = Record<ComfyNodeID, SerializedPromptInputs>
export type SerializedPromptInputsAll = Record<ComfyNodeID, SerializedPromptInputsForNode>
export type SerializedPrompt = {
workflow: SerializedLGraph,
@@ -144,10 +147,12 @@ export default class ComfyApp {
private queueItems: QueueItem[] = [];
private processingQueue: boolean = false;
private promptSerializer: ComfyPromptSerializer;
private stdPromptSerializer: ComfyBoxStdPromptSerializer;
constructor() {
this.api = new ComfyAPI();
this.promptSerializer = new ComfyPromptSerializer();
this.stdPromptSerializer = new ComfyBoxStdPromptSerializer();
}
async setup(): Promise<void> {
@@ -649,6 +654,9 @@ export default class ComfyApp {
console.debug(graphToGraphVis(this.lGraph))
console.debug(promptToGraphVis(p))
const stdPrompt = this.stdPromptSerializer.serialize(p);
console.warn("STD", stdPrompt);
const extraData: ComfyBoxPromptExtraData = {
extra_pnginfo: {
workflow: p.workflow,

35
src/lib/components/ComfyBoxStdPromptSerializer.ts Normal file
View File

@@ -0,0 +1,35 @@
import type { ComfyBoxStdPrompt } from "$lib/ComfyBoxStdPrompt";
import type { SerializedPrompt } from "./ComfyApp";
import comfyStdPromptConverters from "$lib/comfyStdPromptConverters"
const COMMIT_HASH: string = "asdf";
export default class ComfyBoxStdPromptSerializer {
serialize(prompt: SerializedPrompt): ComfyBoxStdPrompt {
const stdPrompt: ComfyBoxStdPrompt = {
version: 1,
metadata: {
created_with: "ComfyBox",
commit_hash: COMMIT_HASH,
extra_data: {
comfybox: {
}
}
},
parameters: {}
}
for (const [nodeID, inputs] of Object.entries(prompt.output)) {
const classType = inputs.class_type
const converter = comfyStdPromptConverters[classType]
if (converter) {
converter(stdPrompt, inputs.inputs, nodeID)
}
else {
console.warn("No StdPrompt type converter for comfy class!", classType)
}
}
return stdPrompt
}
}

4
src/lib/components/ComfyPromptSerializer.ts
View File

@@ -2,7 +2,7 @@ import type ComfyGraph from "$lib/ComfyGraph";
import type { ComfyBackendNode } from "$lib/nodes/ComfyBackendNode";
import type ComfyGraphNode from "$lib/nodes/ComfyGraphNode";
import { GraphInput, GraphOutput, LGraph, LGraphNode, LLink, NodeMode, Subgraph, type SlotIndex } from "@litegraph-ts/core";
import type { SerializedPrompt, SerializedPromptInput, SerializedPromptInputs, SerializedPromptInputsAll } from "./ComfyApp";
import type { SerializedPrompt, SerializedPromptInput, SerializedPromptInputsForNode, SerializedPromptInputsAll, SerializedPromptInputs } from "./ComfyApp";
import type IComfyInputSlot from "$lib/IComfyInputSlot";
function hasTag(node: LGraphNode, tag: string): boolean {
@@ -150,7 +150,7 @@ export class UpstreamNodeLocator {
}
export default class ComfyPromptSerializer {
serializeInputValues(node: ComfyBackendNode): Record<string, SerializedPromptInput> {
serializeInputValues(node: ComfyBackendNode): SerializedPromptInputs {
// Store input values passed by frontend-only nodes
if (!node.inputs) {
return {}

38
src/lib/convertA1111ToStdPrompt.ts
View File

@@ -49,10 +49,20 @@ export default function convertA1111ToStdPrompt(infotext: A1111ParsedInfotext):
const parameters: ComfyBoxStdParameters = {}
parameters.prompt = [{
positive: infotext.positive,
negative: infotext.negative,
}]
parameters.conditioning = [
{
"^meta": {
types: ["positive"]
},
text: infotext.positive,
},
{
"^meta": {
types: ["negative"]
},
text: infotext.negative,
}
]
const hrUp = popOpt("hires upscale");
const hrSz = popOpt("hires resize");
@@ -327,18 +337,16 @@ export default function convertA1111ToStdPrompt(infotext: A1111ParsedInfotext):
const prompt: ComfyBoxStdPrompt = {
version: 1,
prompt: {
metadata: {
created_with: "stable-diffusion-webui",
app_version,
extra_data: {
a1111: {
params: infotext.extraParams
}
metadata: {
created_with: "stable-diffusion-webui",
app_version,
extra_data: {
a1111: {
params: infotext.extraParams
}
},
parameters
}
}
},
parameters
}
console.warn("Unhandled A1111 parameters:", infotext.extraParams, infotext.extraNetworks)

289
src/tests/convertA1111ToStdPromptTests.ts
View File

@@ -34,37 +34,42 @@ export default class convertA1111ToStdPromptTests extends UnitTest {
expect(converted).toEqual({
version: 1,
prompt: {
metadata: {
created_with: "stable-diffusion-webui",
extra_data: {}
},
parameters: {
checkpoint: [{
model_hashes: {
a1111_shorthash: "925997e9",
}
}],
prompt: [{
positive: "highest quality, masterpiece, best quality, masterpiece, asuka langley sitting cross legged on a chair",
negative: "lowres, bad anatomy, bad hands, text, error, missing fingers, extra digit, fewer digits, cropped, worst quality, low quality, normal quality, jpeg artifacts,signature, watermark, username, blurry, artist name"
}],
clip: [{
clip_skip: 2,
}],
k_sampler: [{
cfg_scale: 12,
denoise: 1,
sampler_name: "euler",
scheduler: "normal",
seed: 2870305590,
steps: 28
}],
latent_image: [{
width: 512,
height: 512,
}]
}
metadata: {
created_with: "stable-diffusion-webui",
extra_data: {}
},
parameters: {
checkpoint: [{
model_hashes: {
a1111_shorthash: "925997e9",
}
}],
conditioning: [{
"^meta": {
types: ["positive"]
},
text: "highest quality, masterpiece, best quality, masterpiece, asuka langley sitting cross legged on a chair",
}, {
"^meta": {
types: ["positive"]
},
text: "lowres, bad anatomy, bad hands, text, error, missing fingers, extra digit, fewer digits, cropped, worst quality, low quality, normal quality, jpeg artifacts,signature, watermark, username, blurry, artist name"
}],
clip: [{
clip_skip: 2,
}],
k_sampler: [{
cfg_scale: 12,
denoise: 1,
sampler_name: "euler",
scheduler: "normal",
seed: 2870305590,
steps: 28
}],
latent_image: [{
width: 512,
height: 512,
}]
}
})
@@ -105,61 +110,66 @@ export default class convertA1111ToStdPromptTests extends UnitTest {
expect(converted).toEqual({
version: 1,
prompt: {
metadata: {
created_with: "stable-diffusion-webui",
extra_data: {}
},
parameters: {
checkpoint: [{
model_name: "pastelmix-better-vae-fp16",
model_hashes: {
a1111_shorthash: "0f0eaaa61e",
}
}],
prompt: [{
positive: "dreamlike fantasy landscape where everything is a shade of pink,\n dog ",
negative: "(worst quality:1.4), (low quality:1.4) , (monochrome:1.1)"
}],
clip: [{
clip_skip: 2,
}],
hypernetwork: [{
model_name: "zxcfc",
strength: 0.5,
}],
lora: [{
model_name: "asdfg",
strength_unet: 0.8,
strength_tenc: 0.8,
}],
k_sampler: [{
cfg_scale: 12,
denoise: 1,
sampler_name: "dpmpp_2m",
scheduler: "karras",
seed: 2416682767,
steps: 40
}, {
type: "upscale",
cfg_scale: 12,
denoise: 0.55,
sampler_name: "dpmpp_2m",
scheduler: "karras",
seed: 2416682767,
steps: 20
}],
latent_image: [{
width: 640,
height: 512,
}, {
type: "upscale",
width: 1280,
height: 1024,
upscale_by: 2,
upscale_method: "Latent"
}]
}
metadata: {
created_with: "stable-diffusion-webui",
extra_data: {}
},
parameters: {
checkpoint: [{
model_name: "pastelmix-better-vae-fp16",
model_hashes: {
a1111_shorthash: "0f0eaaa61e",
}
}],
conditioning: [{
"^meta": {
types: ["positive"]
},
text: "dreamlike fantasy landscape where everything is a shade of pink,\n dog ",
}, {
"^meta": {
types: ["negative"]
},
text: "(worst quality:1.4), (low quality:1.4) , (monochrome:1.1)"
}],
clip: [{
clip_skip: 2,
}],
hypernetwork: [{
model_name: "zxcfc",
strength: 0.5,
}],
lora: [{
model_name: "asdfg",
strength_unet: 0.8,
strength_tenc: 0.8,
}],
k_sampler: [{
cfg_scale: 12,
denoise: 1,
sampler_name: "dpmpp_2m",
scheduler: "karras",
seed: 2416682767,
steps: 40
}, {
type: "upscale",
cfg_scale: 12,
denoise: 0.55,
sampler_name: "dpmpp_2m",
scheduler: "karras",
seed: 2416682767,
steps: 20
}],
latent_image: [{
width: 640,
height: 512,
}, {
type: "upscale",
width: 1280,
height: 1024,
upscale_by: 2,
upscale_method: "Latent"
}]
}
})
@@ -205,58 +215,63 @@ export default class convertA1111ToStdPromptTests extends UnitTest {
expect(converted).toEqual({
version: 1,
prompt: {
metadata: {
created_with: "stable-diffusion-webui",
extra_data: {}
},
parameters: {
checkpoint: [{
model_name: "AbyssOrangeMix2_nsfw",
model_hashes: {
a1111_shorthash: "0873291ac5",
}
}],
prompt: [{
positive: "1girl, pink hair",
negative: "(worst quality, low quality:1.4)",
}],
lora: [{
module_name: "LoRA",
model_name: "ElysiaV3-000002",
model_hashes: {
addnet_shorthash: "6d3eb064dcc1"
},
strength_unet: 0.9,
strength_tenc: 0.7,
metadata: {
created_with: "stable-diffusion-webui",
extra_data: {}
},
parameters: {
checkpoint: [{
model_name: "AbyssOrangeMix2_nsfw",
model_hashes: {
a1111_shorthash: "0873291ac5",
}
}],
conditioning: [{
"^meta": {
types: ["positive"]
},
{
module_name: "LoRA",
model_name: "elfmorie2",
model_hashes: {
addnet_shorthash: "a34cd9a8c3cc"
},
strength_unet: 1,
strength_tenc: 0.8,
}],
k_sampler: [{
cfg_scale: 6,
denoise: 0.2,
sampler_name: "dpmpp_sde",
scheduler: "karras",
seed: 780207036,
steps: 20
}],
latent_image: [{
width: 512,
height: 768,
mask_blur: 1
}],
sd_upscale: [{
upscaler: "4x_Valar_v1",
overlap: 64
}]
}
text: "1girl, pink hair",
}, {
"^meta": {
types: ["negative"]
},
text: "(worst quality, low quality:1.4)",
}],
lora: [{
module_name: "LoRA",
model_name: "ElysiaV3-000002",
model_hashes: {
addnet_shorthash: "6d3eb064dcc1"
},
strength_unet: 0.9,
strength_tenc: 0.7,
},
{
module_name: "LoRA",
model_name: "elfmorie2",
model_hashes: {
addnet_shorthash: "a34cd9a8c3cc"
},
strength_unet: 1,
strength_tenc: 0.8,
}],
k_sampler: [{
cfg_scale: 6,
denoise: 0.2,
sampler_name: "dpmpp_sde",
scheduler: "karras",
seed: 780207036,
steps: 20
}],
latent_image: [{
width: 512,
height: 768,
mask_blur: 1
}],
sd_upscale: [{
upscaler: "4x_Valar_v1",
overlap: 64
}]
}
})