From ec80884684c4f3b1c7ec414f9489ca222ef038a5 Mon Sep 17 00:00:00 2001 From: space-nuko <24979496+space-nuko@users.noreply.github.com> Date: Fri, 19 May 2023 11:05:35 -0500 Subject: [PATCH] Conversion to standard prompt format --- package.json | 2 +- pnpm-lock.yaml | 140 ++++++--- src/lib/ComfyBoxStdPrompt.ts | 78 ++++- src/lib/api.ts | 2 +- src/lib/comfyStdPromptConverters.ts | 26 ++ src/lib/components/ComfyApp.ts | 14 +- .../components/ComfyBoxStdPromptSerializer.ts | 35 +++ src/lib/components/ComfyPromptSerializer.ts | 4 +- src/lib/convertA1111ToStdPrompt.ts | 38 ++- src/tests/convertA1111ToStdPromptTests.ts | 289 +++++++++--------- 10 files changed, 404 insertions(+), 224 deletions(-) create mode 100644 src/lib/comfyStdPromptConverters.ts create mode 100644 src/lib/components/ComfyBoxStdPromptSerializer.ts diff --git a/package.json b/package.json index aef40d0..b4545f6 100644 --- a/package.json +++ b/package.json @@ -33,7 +33,7 @@ "svelte-check": "^3.2.0", "svelte-dnd-action": "^0.9.22", "typescript": "^5.0.3", - "vite": "^4.3.1", + "vite": "^4.3.8", "vite-plugin-glsl": "^1.1.2", "vite-plugin-static-copy": "^0.14.0", "vite-plugin-svelte-console-remover": "^1.0.10", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d1bd089..50abc05 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -69,7 +69,7 @@ importers: version: link:litegraph/packages/tsconfig '@sveltejs/vite-plugin-svelte': specifier: ^2.1.1 - version: 2.1.1(svelte@3.58.0)(vite@4.3.1) + version: 2.1.1(svelte@3.58.0)(vite@4.3.8) '@tsconfig/svelte': specifier: ^4.0.1 version: 4.0.1 @@ -123,7 +123,7 @@ importers: version: 9.0.0 vite-plugin-full-reload: specifier: ^1.0.5 - version: 1.0.5(vite@4.3.1) + version: 1.0.5(vite@4.3.8) zod: specifier: ^3.21.4 version: 3.21.4 @@ -174,20 +174,20 @@ importers: specifier: ^5.0.3 version: 5.0.3 vite: - specifier: ^4.3.1 - version: 4.3.1(sass@1.61.0) + specifier: ^4.3.8 + version: 4.3.8(sass@1.61.0) vite-plugin-glsl: specifier: ^1.1.2 - version: 1.1.2(vite@4.3.1) + version: 1.1.2(vite@4.3.8) vite-plugin-static-copy: specifier: ^0.14.0 - version: 0.14.0(vite@4.3.1) + version: 0.14.0(vite@4.3.8) vite-plugin-svelte-console-remover: specifier: ^1.0.10 version: 1.0.10(sass@1.61.0) vite-tsconfig-paths: specifier: ^4.0.8 - version: 4.0.8(typescript@5.0.3)(vite@4.3.1) + version: 4.0.8(typescript@5.0.3)(vite@4.3.8) vitest: specifier: ^0.27.3 version: 0.27.3(happy-dom@9.18.3)(jsdom@22.0.0)(sass@1.61.0) @@ -3044,7 +3044,7 @@ packages: - supports-color dev: true - /@sveltejs/vite-plugin-svelte@2.1.1(svelte@3.58.0)(vite@4.3.1): + /@sveltejs/vite-plugin-svelte@2.1.1(svelte@3.58.0)(vite@4.3.8): resolution: {integrity: sha512-7YeBDt4us0FiIMNsVXxyaP4Hwyn2/v9x3oqStkHU3ZdIc5O22pGwUwH33wUqYo+7Itdmo8zxJ45Qvfm3H7UUjQ==} engines: {node: ^14.18.0 || >= 16} peerDependencies: @@ -3057,8 +3057,8 @@ packages: magic-string: 0.30.0 svelte: 3.58.0 svelte-hmr: 0.15.1(svelte@3.58.0) - vite: 4.3.1(sass@1.61.0) - vitefu: 0.2.4(vite@4.3.1) + vite: 4.3.8(sass@1.61.0) + vitefu: 0.2.4(vite@4.3.8) transitivePeerDependencies: - supports-color dev: false @@ -3066,7 +3066,7 @@ packages: /@swc/helpers@0.4.14: resolution: {integrity: sha512-4C7nX/dvpzB7za4Ql9K81xK3HPxCpHMgwTZVyf+9JQ6VUbn9jjZVN7/Nkdz/Ugzs2CSjqnL/UPXroiVBVHUWUw==} dependencies: - tslib: 2.5.0 + tslib: 2.5.1 dev: false /@tootallnate/once@2.0.0: @@ -3528,7 +3528,7 @@ packages: engines: {node: '>=16.1.0'} dependencies: '@babel/runtime': 7.21.0 - tslib: 2.5.0 + tslib: 2.5.1 dev: false /autoprefixer@10.4.2(postcss@8.4.21): @@ -3700,7 +3700,7 @@ packages: dependencies: '@babel/runtime': 7.21.0 fast-unique-numbers: 8.0.0 - tslib: 2.5.0 + tslib: 2.5.1 worker-factory: 7.0.0 dev: false @@ -3975,7 +3975,7 @@ packages: '@babel/runtime': 7.21.0 dashify: 2.0.0 indefinite-article: 0.0.2 - tslib: 2.5.0 + tslib: 2.5.1 dev: false /compilerr@11.0.0: @@ -3985,7 +3985,7 @@ packages: '@babel/runtime': 7.21.0 dashify: 2.0.0 indefinite-article: 0.0.2 - tslib: 2.5.0 + tslib: 2.5.1 dev: false /concat-map@0.0.1: @@ -4972,14 +4972,14 @@ packages: '@babel/runtime': 7.21.0 broker-factory: 3.0.76 extendable-media-recorder-wav-encoder-worker: 8.0.77 - tslib: 2.5.0 + tslib: 2.5.1 dev: false /extendable-media-recorder-wav-encoder-worker@8.0.77: resolution: {integrity: sha512-9g9Q7fhOxPY7RALHVTK9Wjnc8RPYjJ9XCBP1TaNtDraIAFxvhBRax9QUOmFqHM2MvRM6hQhNav7jn23yy6tcVQ==} dependencies: '@babel/runtime': 7.21.0 - tslib: 2.5.0 + tslib: 2.5.1 worker-factory: 7.0.0 dev: false @@ -5042,7 +5042,7 @@ packages: engines: {node: '>=14.15.4'} dependencies: '@babel/runtime': 7.21.0 - tslib: 2.5.0 + tslib: 2.5.1 dev: false /fast-unique-numbers@8.0.0: @@ -5050,7 +5050,7 @@ packages: engines: {node: '>=16.1.0'} dependencies: '@babel/runtime': 7.21.0 - tslib: 2.5.0 + tslib: 2.5.1 dev: false /fastq@1.15.0: @@ -6646,7 +6646,7 @@ packages: broker-factory: 3.0.76 fast-unique-numbers: 8.0.0 media-encoder-host-worker: 9.1.1 - tslib: 2.5.0 + tslib: 2.5.1 dev: false /media-encoder-host-worker@9.1.1: @@ -6654,7 +6654,7 @@ packages: dependencies: '@babel/runtime': 7.21.0 extendable-media-recorder-wav-encoder-broker: 7.0.78 - tslib: 2.5.0 + tslib: 2.5.1 worker-factory: 7.0.0 dev: false @@ -6664,7 +6664,7 @@ packages: '@babel/runtime': 7.21.0 media-encoder-host-broker: 7.0.79 media-encoder-host-worker: 9.1.1 - tslib: 2.5.0 + tslib: 2.5.1 dev: false /merge-stream@2.0.0: @@ -6782,7 +6782,7 @@ packages: engines: {node: '>=12.20.1'} dependencies: '@babel/runtime': 7.21.0 - tslib: 2.5.0 + tslib: 2.5.1 dev: false /murmurhash-js@1.0.0: @@ -7198,6 +7198,14 @@ packages: picocolors: 1.0.0 source-map-js: 1.0.2 + /postcss@8.4.23: + resolution: {integrity: sha512-bQ3qMcpF6A/YjR55xtoTr0jGOlnPOKAIMdOWiv0EIT6HVPEaJiJB4NLljSbiHoC2RX7DN5Uvjtpbg1NPdwv1oA==} + engines: {node: ^10 || ^12 || >=14} + dependencies: + nanoid: 3.3.6 + picocolors: 1.0.0 + source-map-js: 1.0.2 + /posthtml-parser@0.10.2: resolution: {integrity: sha512-PId6zZ/2lyJi9LiKfe+i2xv57oEjJgWbsHGGANwos5AvdQp98i6AtamAl8gzSVFGfQ43Glb5D614cvZf012VKg==} engines: {node: '>=12'} @@ -7382,7 +7390,7 @@ packages: resolution: {integrity: sha512-oiiS2sp6eMxkvjt13yetSYUJvnAxBZk60mIxz0Vf/2lDWa/4svCyMLHIDzYKbHahkISd0UYyqLS9dI7xDlUOCA==} dependencies: '@babel/runtime': 7.21.0 - tslib: 2.5.0 + tslib: 2.5.1 dev: false /recorder-audio-worklet@5.1.39: @@ -7394,7 +7402,7 @@ packages: recorder-audio-worklet-processor: 4.2.21 standardized-audio-context: 25.3.45 subscribable-things: 2.1.14 - tslib: 2.5.0 + tslib: 2.5.1 worker-factory: 6.0.76 dev: false @@ -7777,7 +7785,7 @@ packages: dependencies: '@babel/runtime': 7.21.0 automation-events: 6.0.0 - tslib: 2.5.0 + tslib: 2.5.1 dev: false /std-env@3.3.3: @@ -7861,7 +7869,7 @@ packages: dependencies: '@babel/runtime': 7.21.0 rxjs-interop: 2.0.0 - tslib: 2.5.0 + tslib: 2.5.1 dev: false /sucrase@3.32.0: @@ -8450,7 +8458,6 @@ packages: /tslib@2.5.1: resolution: {integrity: sha512-KaI6gPil5m9vF7DKaoXxx1ia9fxS4qG5YveErRRVknPDXXriu5M8h48YRjB6h5ZUOKuAKlSJYb0GaDe8I39fRw==} dev: false - optional: true /tsutils@3.21.0(typescript@5.0.3): resolution: {integrity: sha512-mHKK3iUXL+3UF6xL5k0PEhKRUBKPBCv/+RkEOpjRWxxx27KKRBmmA60A9pgOUvMi8GKhRMPEmjBRPzs2W7O1OA==} @@ -8629,7 +8636,7 @@ packages: fast-json-patch: 3.1.1 json-stringify-pretty-compact: 3.0.0 semver: 7.4.0 - tslib: 2.5.0 + tslib: 2.5.1 vega: 5.22.1 vega-interpreter: 1.0.5 vega-lite: 0.6.7 @@ -9007,7 +9014,7 @@ packages: picocolors: 1.0.0 source-map: 0.6.1 source-map-support: 0.5.21 - vite: 4.3.1(@types/node@18.16.0)(sass@1.61.0) + vite: 4.3.8(@types/node@18.16.0)(sass@1.61.0) transitivePeerDependencies: - '@types/node' - less @@ -9028,7 +9035,7 @@ packages: mlly: 1.2.1 pathe: 1.1.0 picocolors: 1.0.0 - vite: 4.3.1(@types/node@18.16.0) + vite: 4.3.8(@types/node@18.16.0) transitivePeerDependencies: - '@types/node' - less @@ -9114,29 +9121,29 @@ packages: - supports-color dev: false - /vite-plugin-full-reload@1.0.5(vite@4.3.1): + /vite-plugin-full-reload@1.0.5(vite@4.3.8): resolution: {integrity: sha512-kVZFDFWr0DxiHn6MuDVTQf7gnWIdETGlZh0hvTiMXzRN80vgF4PKbONSq8U1d0WtHsKaFODTQgJeakLacoPZEQ==} peerDependencies: vite: ^2 || ^3 || ^4 dependencies: picocolors: 1.0.0 picomatch: 2.3.1 - vite: 4.3.1(sass@1.61.0) + vite: 4.3.8(sass@1.61.0) dev: false - /vite-plugin-glsl@1.1.2(vite@4.3.1): + /vite-plugin-glsl@1.1.2(vite@4.3.8): resolution: {integrity: sha512-zmXsfc1vn2MlYve9t3FAoWuhLyoCkNS1TuQL+TkXZL7tGmBjRErp10eNYxcse5tK9oUC5MyJpNc4ElpQnx8DoA==} engines: {node: '>= 16.15.1', npm: '>= 8.11.0'} peerDependencies: vite: ^3.0.0 || ^4.0.0 dependencies: '@rollup/pluginutils': 5.0.2 - vite: 4.3.1(sass@1.61.0) + vite: 4.3.8(sass@1.61.0) transitivePeerDependencies: - rollup dev: true - /vite-plugin-static-copy@0.14.0(vite@4.3.1): + /vite-plugin-static-copy@0.14.0(vite@4.3.8): resolution: {integrity: sha512-RMFmb4czomcrsbQBiUZs9HcDGN3kxGvF+OrtkfTVocp12CuoUCuJQhcY26RK35A6KS4WasGzEwcYZqHMjkAvVw==} engines: {node: ^14.18.0 || >=16.0.0} peerDependencies: @@ -9146,7 +9153,7 @@ packages: fast-glob: 3.2.12 fs-extra: 11.1.1 picocolors: 1.0.0 - vite: 4.3.1(sass@1.61.0) + vite: 4.3.8(sass@1.61.0) dev: true /vite-plugin-svelte-console-remover@1.0.10(sass@1.61.0): @@ -9162,7 +9169,7 @@ packages: - supports-color dev: true - /vite-tsconfig-paths@4.0.8(typescript@5.0.3)(vite@4.3.1): + /vite-tsconfig-paths@4.0.8(typescript@5.0.3)(vite@4.3.8): resolution: {integrity: sha512-p04zH+Ey+NT78571x0pdX7nVRIJSlmKVvYryFglSWOK3Hc72eDL0+JJfbyQiugaIBApJkaEqbBQvqpsFZOSVGg==} peerDependencies: vite: '*' @@ -9173,7 +9180,7 @@ packages: debug: 4.3.4 globrex: 0.1.2 tsconfck: 2.1.1(typescript@5.0.3) - vite: 4.3.1(sass@1.61.0) + vite: 4.3.8(sass@1.61.0) transitivePeerDependencies: - supports-color - typescript @@ -9325,8 +9332,8 @@ packages: fsevents: 2.3.2 dev: false - /vite@4.3.1(@types/node@18.16.0)(sass@1.61.0): - resolution: {integrity: sha512-EPmfPLAI79Z/RofuMvkIS0Yr091T2ReUoXQqc5ppBX/sjFRhHKiPPF/R46cTdoci/XgeQpB23diiJxq5w30vdg==} + /vite@4.3.8(@types/node@18.16.0): + resolution: {integrity: sha512-uYB8PwN7hbMrf4j1xzGDk/lqjsZvCDbt/JC5dyfxc19Pg8kRm14LinK/uq+HSLNswZEoKmweGdtpbnxRtrAXiQ==} engines: {node: ^14.18.0 || >=16.0.0} hasBin: true peerDependencies: @@ -9352,15 +9359,48 @@ packages: dependencies: '@types/node': 18.16.0 esbuild: 0.17.18 - postcss: 8.4.21 + postcss: 8.4.23 + rollup: 3.21.0 + optionalDependencies: + fsevents: 2.3.2 + dev: false + + /vite@4.3.8(@types/node@18.16.0)(sass@1.61.0): + resolution: {integrity: sha512-uYB8PwN7hbMrf4j1xzGDk/lqjsZvCDbt/JC5dyfxc19Pg8kRm14LinK/uq+HSLNswZEoKmweGdtpbnxRtrAXiQ==} + engines: {node: ^14.18.0 || >=16.0.0} + hasBin: true + peerDependencies: + '@types/node': '>= 14' + less: '*' + sass: '*' + stylus: '*' + sugarss: '*' + terser: ^5.4.0 + peerDependenciesMeta: + '@types/node': + optional: true + less: + optional: true + sass: + optional: true + stylus: + optional: true + sugarss: + optional: true + terser: + optional: true + dependencies: + '@types/node': 18.16.0 + esbuild: 0.17.18 + postcss: 8.4.23 rollup: 3.21.0 sass: 1.61.0 optionalDependencies: fsevents: 2.3.2 dev: true - /vite@4.3.1(sass@1.61.0): - resolution: {integrity: sha512-EPmfPLAI79Z/RofuMvkIS0Yr091T2ReUoXQqc5ppBX/sjFRhHKiPPF/R46cTdoci/XgeQpB23diiJxq5w30vdg==} + /vite@4.3.8(sass@1.61.0): + resolution: {integrity: sha512-uYB8PwN7hbMrf4j1xzGDk/lqjsZvCDbt/JC5dyfxc19Pg8kRm14LinK/uq+HSLNswZEoKmweGdtpbnxRtrAXiQ==} engines: {node: ^14.18.0 || >=16.0.0} hasBin: true peerDependencies: @@ -9385,7 +9425,7 @@ packages: optional: true dependencies: esbuild: 0.17.18 - postcss: 8.4.21 + postcss: 8.4.23 rollup: 3.21.0 sass: 1.61.0 optionalDependencies: @@ -9400,7 +9440,7 @@ packages: optional: true dev: true - /vitefu@0.2.4(vite@4.3.1): + /vitefu@0.2.4(vite@4.3.8): resolution: {integrity: sha512-fanAXjSaf9xXtOOeno8wZXIhgia+CZury481LsDaV++lSvcU2R9Ch2bPh3PYFyoHW+w9LqAeYRISVQjUIew14g==} peerDependencies: vite: ^3.0.0 || ^4.0.0 @@ -9408,7 +9448,7 @@ packages: vite: optional: true dependencies: - vite: 4.3.1(sass@1.61.0) + vite: 4.3.8(sass@1.61.0) dev: false /vitest@0.27.3(happy-dom@9.18.3)(jsdom@22.0.0)(sass@1.61.0): @@ -9451,7 +9491,7 @@ packages: tinybench: 2.5.0 tinypool: 0.3.1 tinyspy: 1.1.1 - vite: 4.3.1(@types/node@18.16.0)(sass@1.61.0) + vite: 4.3.8(@types/node@18.16.0)(sass@1.61.0) vite-node: 0.27.3(@types/node@18.16.0)(sass@1.61.0) why-is-node-running: 2.2.2 transitivePeerDependencies: @@ -9657,7 +9697,7 @@ packages: '@babel/runtime': 7.21.0 compilerr: 10.0.2 fast-unique-numbers: 7.0.2 - tslib: 2.5.0 + tslib: 2.5.1 dev: false /worker-factory@7.0.0: @@ -9666,7 +9706,7 @@ packages: '@babel/runtime': 7.21.0 compilerr: 11.0.0 fast-unique-numbers: 8.0.0 - tslib: 2.5.0 + tslib: 2.5.1 dev: false /wrap-ansi@7.0.0: diff --git a/src/lib/ComfyBoxStdPrompt.ts b/src/lib/ComfyBoxStdPrompt.ts index ac44ed7..bbb8b75 100644 --- a/src/lib/ComfyBoxStdPrompt.ts +++ b/src/lib/ComfyBoxStdPrompt.ts @@ -7,11 +7,10 @@ const ModelHashes = z.object({ a1111_shorthash !== undefined || sha256 !== undefined, { message: "At least one model hash must be specified" }) -const GroupPrompt = z.object({ - positive: z.string(), - negative: z.string() +const GroupConditioning = z.object({ + text: z.string(), }) -export type ComfyBoxStdGroupPrompt = z.infer +export type ComfyBoxStdGroupConditioning = z.infer const GroupCheckpoint = z.object({ model_name: z.string().optional(), @@ -138,15 +137,46 @@ const GroupDDetailer = z.object({ dilation: z.number(), offset_x: z.number(), offset_y: z.number(), - inpaint_full: z.number(), + preprocess: z.boolean(), + inpaint_full: z.boolean(), inpaint_padding: z.number(), + cfg: z.number() }) export type ComfyBoxStdGroupDDetailer = z.infer -const group = (s: ZodTypeAny) => z.optional(z.array(s).nonempty()); +/* + * This metadata can be attached to each entry in a group to assist in + * identifying the correct nodes to apply it to. + * + * As an example, positive and negative conditioning are deployed as two + * separate nodes in ComfyUI. This makes bundling them into a { positive, + * negative } entry difficult as either one can be missing. So instead they're + * tagged like + * + * { + * conditioning: [ + * { text: "masterpiece", "$meta": { types: ["positive"] } }, + * { text: "worst quality", "$meta": { types: ["negative"] } }, + * ] + * } + * + * The reasoning is the "types" information isn't required to reinstantiate + * the node, it's only semantic information describing how the node is used in + * the encompassing workflow. When the prompt is loaded the workflow can be + * searched for a node with the compatible type to attach the information to. + */ +const GroupMetadata = z.object({ + types: z.array(z.string()).nonempty().optional() +}) +export type ComfyBoxStdGroupMetadata = z.infer + +const group = (entry: ZodTypeAny) => { + const groupEntry = entry.and(z.object({ "$meta": GroupMetadata })) + return z.optional(z.array(groupEntry).nonempty()); +} const Parameters = z.object({ - prompt: group(GroupPrompt), + conditioning: group(GroupConditioning), checkpoint: group(GroupCheckpoint), vae: group(GroupVAE), k_sampler: group(GroupKSampler), @@ -183,22 +213,40 @@ const Metadata = z.object({ extra_data: ExtraData }) -const Prompt = z.object({ - metadata: Metadata, - parameters: Parameters -}) - const ComfyBoxStdPrompt = z.object({ version: z.number(), - prompt: Prompt, + metadata: Metadata, + parameters: Parameters }) export default ComfyBoxStdPrompt /* - * A standardized Stable Diffusion prompt and parameter format, to be used with - * an encompassing workflow. Aims to encompass an arbitrary number of parameter + * A standardized Stable Diffusion parameter format that should be used with an + * encompassing workflow. Aims to encompass an arbitrary number of parameter * counts and types, so that most ComfyUI workflows can have parts of their * prompts transferred between each other. + * + * This format does *not* describe how the information should be used in the + * underlying workflow, i.e. it does not specify the structure of a ComfyUI + * execution graph. It only gives hints via tagged input types on each input + * entry as to where the data should be inserted. To recreate a ComfyBox + * workflow with the exact state of the UI intact, the `SerializedAppState` type + * should be used instead. It suffices to embed data of that type in the output + * PNGs for recreating their workflows. This type is meant as an interchange + * format *between* workflows so their inputs can be copied to and from each + * other in a sane-enough manner. (In ComfyBox, copying workflow outputs like + * images to other workflows is handled separately, since this type does not + * retain the actual image data.) + * + * In contrast with a serialized workflow, which is concerned with the + * connections between nodes and the state of the frontend's UI, this format + * concerns itself with the exact values that the execution backend receives, + * after the data in the UI have finished processing. + * + * (Take for example a "scale by" slider that adjusts the width and height of an + * img2img input image of 512 x 512 resolution by 2x. The backend will only + * "see" width 1024 and height 1024, even though the only parameter exposed from + * the frontend was the scale of 2.) */ export type ComfyBoxStdPrompt = z.infer diff --git a/src/lib/api.ts b/src/lib/api.ts index 8e5097c..78ea676 100644 --- a/src/lib/api.ts +++ b/src/lib/api.ts @@ -1,4 +1,4 @@ -import type { Progress, SerializedPrompt, SerializedPromptInputs, SerializedPromptInputsAll, SerializedPromptOutputs } from "./components/ComfyApp"; +import type { Progress, SerializedPrompt, SerializedPromptInputsForNode, SerializedPromptInputsAll, SerializedPromptOutputs } from "./components/ComfyApp"; import type TypedEmitter from "typed-emitter"; import EventEmitter from "events"; import type { ComfyImageLocation } from "$lib/utils"; diff --git a/src/lib/comfyStdPromptConverters.ts b/src/lib/comfyStdPromptConverters.ts new file mode 100644 index 0000000..5148412 --- /dev/null +++ b/src/lib/comfyStdPromptConverters.ts @@ -0,0 +1,26 @@ +import type { ComfyBoxStdGroupLoRA, ComfyBoxStdPrompt } from "./ComfyBoxStdPrompt"; +import type { ComfyNodeID } from "./api"; +import type { SerializedPromptInputs } from "./components/ComfyApp"; +import type { ComfyBackendNode } from "./nodes/ComfyBackendNode"; + +export type ComfyPromptConverter = (stdPrompt: ComfyBoxStdPrompt, inputs: SerializedPromptInputs, nodeID: ComfyNodeID) => void; + +function LoraLoader(stdPrompt: ComfyBoxStdPrompt, inputs: SerializedPromptInputs) { + const params = stdPrompt.prompt.parameters + + const lora: ComfyBoxStdGroupLoRA = { + model_name: inputs["lora_name"], + strength_unet: inputs["strength_model"], + strength_tenc: inputs["strength_clip"] + } + + if (params.lora) + params.lora.push(lora) + else + params.lora = [lora] +} + +const converters: Record = { + LoraLoader +} +export default converters; diff --git a/src/lib/components/ComfyApp.ts b/src/lib/components/ComfyApp.ts index 49466eb..941168b 100644 --- a/src/lib/components/ComfyApp.ts +++ b/src/lib/components/ComfyApp.ts @@ -40,6 +40,7 @@ import { ComfyComboNode } from "$lib/nodes/widgets"; import parseA1111, { type A1111ParsedInfotext } from "$lib/parseA1111"; import convertA1111ToStdPrompt from "$lib/convertA1111ToStdPrompt"; import type { ComfyBoxStdPrompt } from "$lib/ComfyBoxStdPrompt"; +import ComfyBoxStdPromptSerializer from "./ComfyBoxStdPromptSerializer"; export const COMFYBOX_SERIAL_VERSION = 1; @@ -86,19 +87,21 @@ export type SerializedAppState = { /** [link_origin, link_slot_index] | input_value */ export type SerializedPromptInput = [ComfyNodeID, number] | any +export type SerializedPromptInputs = Record; + /* * A single node in the prompt and its input values. */ -export type SerializedPromptInputs = { +export type SerializedPromptInputsForNode = { /* property name -> value or link */ - inputs: Record, + inputs: SerializedPromptInputs, class_type: string } /* * All nodes in the graph and their input values. */ -export type SerializedPromptInputsAll = Record +export type SerializedPromptInputsAll = Record export type SerializedPrompt = { workflow: SerializedLGraph, @@ -144,10 +147,12 @@ export default class ComfyApp { private queueItems: QueueItem[] = []; private processingQueue: boolean = false; private promptSerializer: ComfyPromptSerializer; + private stdPromptSerializer: ComfyBoxStdPromptSerializer; constructor() { this.api = new ComfyAPI(); this.promptSerializer = new ComfyPromptSerializer(); + this.stdPromptSerializer = new ComfyBoxStdPromptSerializer(); } async setup(): Promise { @@ -649,6 +654,9 @@ export default class ComfyApp { console.debug(graphToGraphVis(this.lGraph)) console.debug(promptToGraphVis(p)) + const stdPrompt = this.stdPromptSerializer.serialize(p); + console.warn("STD", stdPrompt); + const extraData: ComfyBoxPromptExtraData = { extra_pnginfo: { workflow: p.workflow, diff --git a/src/lib/components/ComfyBoxStdPromptSerializer.ts b/src/lib/components/ComfyBoxStdPromptSerializer.ts new file mode 100644 index 0000000..b1d145f --- /dev/null +++ b/src/lib/components/ComfyBoxStdPromptSerializer.ts @@ -0,0 +1,35 @@ +import type { ComfyBoxStdPrompt } from "$lib/ComfyBoxStdPrompt"; +import type { SerializedPrompt } from "./ComfyApp"; +import comfyStdPromptConverters from "$lib/comfyStdPromptConverters" + +const COMMIT_HASH: string = "asdf"; + +export default class ComfyBoxStdPromptSerializer { + serialize(prompt: SerializedPrompt): ComfyBoxStdPrompt { + const stdPrompt: ComfyBoxStdPrompt = { + version: 1, + metadata: { + created_with: "ComfyBox", + commit_hash: COMMIT_HASH, + extra_data: { + comfybox: { + } + } + }, + parameters: {} + } + + for (const [nodeID, inputs] of Object.entries(prompt.output)) { + const classType = inputs.class_type + const converter = comfyStdPromptConverters[classType] + if (converter) { + converter(stdPrompt, inputs.inputs, nodeID) + } + else { + console.warn("No StdPrompt type converter for comfy class!", classType) + } + } + + return stdPrompt + } +} diff --git a/src/lib/components/ComfyPromptSerializer.ts b/src/lib/components/ComfyPromptSerializer.ts index 253d40d..de55bf8 100644 --- a/src/lib/components/ComfyPromptSerializer.ts +++ b/src/lib/components/ComfyPromptSerializer.ts @@ -2,7 +2,7 @@ import type ComfyGraph from "$lib/ComfyGraph"; import type { ComfyBackendNode } from "$lib/nodes/ComfyBackendNode"; import type ComfyGraphNode from "$lib/nodes/ComfyGraphNode"; import { GraphInput, GraphOutput, LGraph, LGraphNode, LLink, NodeMode, Subgraph, type SlotIndex } from "@litegraph-ts/core"; -import type { SerializedPrompt, SerializedPromptInput, SerializedPromptInputs, SerializedPromptInputsAll } from "./ComfyApp"; +import type { SerializedPrompt, SerializedPromptInput, SerializedPromptInputsForNode, SerializedPromptInputsAll, SerializedPromptInputs } from "./ComfyApp"; import type IComfyInputSlot from "$lib/IComfyInputSlot"; function hasTag(node: LGraphNode, tag: string): boolean { @@ -150,7 +150,7 @@ export class UpstreamNodeLocator { } export default class ComfyPromptSerializer { - serializeInputValues(node: ComfyBackendNode): Record { + serializeInputValues(node: ComfyBackendNode): SerializedPromptInputs { // Store input values passed by frontend-only nodes if (!node.inputs) { return {} diff --git a/src/lib/convertA1111ToStdPrompt.ts b/src/lib/convertA1111ToStdPrompt.ts index d501ee5..5bebdc1 100644 --- a/src/lib/convertA1111ToStdPrompt.ts +++ b/src/lib/convertA1111ToStdPrompt.ts @@ -49,10 +49,20 @@ export default function convertA1111ToStdPrompt(infotext: A1111ParsedInfotext): const parameters: ComfyBoxStdParameters = {} - parameters.prompt = [{ - positive: infotext.positive, - negative: infotext.negative, - }] + parameters.conditioning = [ + { + "^meta": { + types: ["positive"] + }, + text: infotext.positive, + }, + { + "^meta": { + types: ["negative"] + }, + text: infotext.negative, + } + ] const hrUp = popOpt("hires upscale"); const hrSz = popOpt("hires resize"); @@ -327,18 +337,16 @@ export default function convertA1111ToStdPrompt(infotext: A1111ParsedInfotext): const prompt: ComfyBoxStdPrompt = { version: 1, - prompt: { - metadata: { - created_with: "stable-diffusion-webui", - app_version, - extra_data: { - a1111: { - params: infotext.extraParams - } + metadata: { + created_with: "stable-diffusion-webui", + app_version, + extra_data: { + a1111: { + params: infotext.extraParams } - }, - parameters - } + } + }, + parameters } console.warn("Unhandled A1111 parameters:", infotext.extraParams, infotext.extraNetworks) diff --git a/src/tests/convertA1111ToStdPromptTests.ts b/src/tests/convertA1111ToStdPromptTests.ts index c6854fc..b20dee6 100644 --- a/src/tests/convertA1111ToStdPromptTests.ts +++ b/src/tests/convertA1111ToStdPromptTests.ts @@ -34,37 +34,42 @@ export default class convertA1111ToStdPromptTests extends UnitTest { expect(converted).toEqual({ version: 1, - prompt: { - metadata: { - created_with: "stable-diffusion-webui", - extra_data: {} - }, - parameters: { - checkpoint: [{ - model_hashes: { - a1111_shorthash: "925997e9", - } - }], - prompt: [{ - positive: "highest quality, masterpiece, best quality, masterpiece, asuka langley sitting cross legged on a chair", - negative: "lowres, bad anatomy, bad hands, text, error, missing fingers, extra digit, fewer digits, cropped, worst quality, low quality, normal quality, jpeg artifacts,signature, watermark, username, blurry, artist name" - }], - clip: [{ - clip_skip: 2, - }], - k_sampler: [{ - cfg_scale: 12, - denoise: 1, - sampler_name: "euler", - scheduler: "normal", - seed: 2870305590, - steps: 28 - }], - latent_image: [{ - width: 512, - height: 512, - }] - } + metadata: { + created_with: "stable-diffusion-webui", + extra_data: {} + }, + parameters: { + checkpoint: [{ + model_hashes: { + a1111_shorthash: "925997e9", + } + }], + conditioning: [{ + "^meta": { + types: ["positive"] + }, + text: "highest quality, masterpiece, best quality, masterpiece, asuka langley sitting cross legged on a chair", + }, { + "^meta": { + types: ["positive"] + }, + text: "lowres, bad anatomy, bad hands, text, error, missing fingers, extra digit, fewer digits, cropped, worst quality, low quality, normal quality, jpeg artifacts,signature, watermark, username, blurry, artist name" + }], + clip: [{ + clip_skip: 2, + }], + k_sampler: [{ + cfg_scale: 12, + denoise: 1, + sampler_name: "euler", + scheduler: "normal", + seed: 2870305590, + steps: 28 + }], + latent_image: [{ + width: 512, + height: 512, + }] } }) @@ -105,61 +110,66 @@ export default class convertA1111ToStdPromptTests extends UnitTest { expect(converted).toEqual({ version: 1, - prompt: { - metadata: { - created_with: "stable-diffusion-webui", - extra_data: {} - }, - parameters: { - checkpoint: [{ - model_name: "pastelmix-better-vae-fp16", - model_hashes: { - a1111_shorthash: "0f0eaaa61e", - } - }], - prompt: [{ - positive: "dreamlike fantasy landscape where everything is a shade of pink,\n dog ", - negative: "(worst quality:1.4), (low quality:1.4) , (monochrome:1.1)" - }], - clip: [{ - clip_skip: 2, - }], - hypernetwork: [{ - model_name: "zxcfc", - strength: 0.5, - }], - lora: [{ - model_name: "asdfg", - strength_unet: 0.8, - strength_tenc: 0.8, - }], - k_sampler: [{ - cfg_scale: 12, - denoise: 1, - sampler_name: "dpmpp_2m", - scheduler: "karras", - seed: 2416682767, - steps: 40 - }, { - type: "upscale", - cfg_scale: 12, - denoise: 0.55, - sampler_name: "dpmpp_2m", - scheduler: "karras", - seed: 2416682767, - steps: 20 - }], - latent_image: [{ - width: 640, - height: 512, - }, { - type: "upscale", - width: 1280, - height: 1024, - upscale_by: 2, - upscale_method: "Latent" - }] - } + metadata: { + created_with: "stable-diffusion-webui", + extra_data: {} + }, + parameters: { + checkpoint: [{ + model_name: "pastelmix-better-vae-fp16", + model_hashes: { + a1111_shorthash: "0f0eaaa61e", + } + }], + conditioning: [{ + "^meta": { + types: ["positive"] + }, + text: "dreamlike fantasy landscape where everything is a shade of pink,\n dog ", + }, { + "^meta": { + types: ["negative"] + }, + text: "(worst quality:1.4), (low quality:1.4) , (monochrome:1.1)" + }], + clip: [{ + clip_skip: 2, + }], + hypernetwork: [{ + model_name: "zxcfc", + strength: 0.5, + }], + lora: [{ + model_name: "asdfg", + strength_unet: 0.8, + strength_tenc: 0.8, + }], + k_sampler: [{ + cfg_scale: 12, + denoise: 1, + sampler_name: "dpmpp_2m", + scheduler: "karras", + seed: 2416682767, + steps: 40 + }, { + type: "upscale", + cfg_scale: 12, + denoise: 0.55, + sampler_name: "dpmpp_2m", + scheduler: "karras", + seed: 2416682767, + steps: 20 + }], + latent_image: [{ + width: 640, + height: 512, + }, { + type: "upscale", + width: 1280, + height: 1024, + upscale_by: 2, + upscale_method: "Latent" + }] } }) @@ -205,58 +215,63 @@ export default class convertA1111ToStdPromptTests extends UnitTest { expect(converted).toEqual({ version: 1, - prompt: { - metadata: { - created_with: "stable-diffusion-webui", - extra_data: {} - }, - parameters: { - checkpoint: [{ - model_name: "AbyssOrangeMix2_nsfw", - model_hashes: { - a1111_shorthash: "0873291ac5", - } - }], - prompt: [{ - positive: "1girl, pink hair", - negative: "(worst quality, low quality:1.4)", - }], - lora: [{ - module_name: "LoRA", - model_name: "ElysiaV3-000002", - model_hashes: { - addnet_shorthash: "6d3eb064dcc1" - }, - strength_unet: 0.9, - strength_tenc: 0.7, + metadata: { + created_with: "stable-diffusion-webui", + extra_data: {} + }, + parameters: { + checkpoint: [{ + model_name: "AbyssOrangeMix2_nsfw", + model_hashes: { + a1111_shorthash: "0873291ac5", + } + }], + conditioning: [{ + "^meta": { + types: ["positive"] }, - { - module_name: "LoRA", - model_name: "elfmorie2", - model_hashes: { - addnet_shorthash: "a34cd9a8c3cc" - }, - strength_unet: 1, - strength_tenc: 0.8, - }], - k_sampler: [{ - cfg_scale: 6, - denoise: 0.2, - sampler_name: "dpmpp_sde", - scheduler: "karras", - seed: 780207036, - steps: 20 - }], - latent_image: [{ - width: 512, - height: 768, - mask_blur: 1 - }], - sd_upscale: [{ - upscaler: "4x_Valar_v1", - overlap: 64 - }] - } + text: "1girl, pink hair", + }, { + "^meta": { + types: ["negative"] + }, + text: "(worst quality, low quality:1.4)", + }], + lora: [{ + module_name: "LoRA", + model_name: "ElysiaV3-000002", + model_hashes: { + addnet_shorthash: "6d3eb064dcc1" + }, + strength_unet: 0.9, + strength_tenc: 0.7, + }, + { + module_name: "LoRA", + model_name: "elfmorie2", + model_hashes: { + addnet_shorthash: "a34cd9a8c3cc" + }, + strength_unet: 1, + strength_tenc: 0.8, + }], + k_sampler: [{ + cfg_scale: 6, + denoise: 0.2, + sampler_name: "dpmpp_sde", + scheduler: "karras", + seed: 780207036, + steps: 20 + }], + latent_image: [{ + width: 512, + height: 768, + mask_blur: 1 + }], + sd_upscale: [{ + upscaler: "4x_Valar_v1", + overlap: 64 + }] } })