From eb02561906c2c673f054975e7b199aa8b9bfb2cd Mon Sep 17 00:00:00 2001 From: space-nuko <24979496+space-nuko@users.noreply.github.com> Date: Mon, 5 Jun 2023 15:45:08 -0500 Subject: [PATCH 1/3] Basic preview support (as of latest PR commit) --- src/lib/api.ts | 97 ++++++++++++------- src/lib/components/ComfyApp.ts | 4 + .../components/ComfyBoxWorkflowsView.svelte | 3 + src/lib/nodes/widgets/ComfyGalleryNode.ts | 6 +- src/lib/stores/layoutStates.ts | 8 ++ src/lib/stores/queueState.ts | 37 +++++++ src/lib/stores/uiState.ts | 2 + src/lib/widgets/GalleryWidget.svelte | 77 ++++++++++++++- 8 files changed, 198 insertions(+), 36 deletions(-) diff --git a/src/lib/api.ts b/src/lib/api.ts index 492aa2b..ec54872 100644 --- a/src/lib/api.ts +++ b/src/lib/api.ts @@ -101,6 +101,7 @@ export type ComfyUIPromptExtraData = { } type ComfyAPIEvents = { + // JSON status: (status: ComfyAPIStatusResponse | null, error?: Error | null) => void, progress: (progress: Progress) => void, reconnecting: () => void, @@ -111,6 +112,9 @@ type ComfyAPIEvents = { execution_cached: (promptID: PromptID, nodes: ComfyNodeID[]) => void, execution_interrupted: (error: ComfyInterruptedError) => void, execution_error: (error: ComfyExecutionError) => void, + + // Binary + b_preview: (imageBlob: Blob) => void } export default class ComfyAPI { @@ -126,7 +130,7 @@ export default class ComfyAPI { } /** - * Poll status for colab and other things that don't support websockets. + * Poll status for colab and other things that don't support websockets. */ private pollQueue() { setInterval(async () => { @@ -176,6 +180,7 @@ export default class ComfyAPI { this.socket = new WebSocket( `ws${window.location.protocol === "https:" ? "s" : ""}://${hostname}:${port}/ws${existingSession}` ); + this.socket.binaryType = "arraybuffer"; this.socket.addEventListener("open", () => { opened = true; @@ -204,38 +209,64 @@ export default class ComfyAPI { this.socket.addEventListener("message", (event) => { try { - const msg = JSON.parse(event.data); - switch (msg.type) { - case "status": - if (msg.data.sid) { - this.clientId = msg.data.sid; - sessionStorage["Comfy.SessionId"] = this.clientId; - } - this.eventBus.emit("status", { execInfo: { queueRemaining: msg.data.status.exec_info.queue_remaining } }); - break; - case "progress": - this.eventBus.emit("progress", msg.data as Progress); - break; - case "executing": - this.eventBus.emit("executing", msg.data.prompt_id, msg.data.node); - break; - case "executed": - this.eventBus.emit("executed", msg.data.prompt_id, msg.data.node, msg.data.output); - break; - case "execution_start": - this.eventBus.emit("execution_start", msg.data.prompt_id); - break; - case "execution_cached": - this.eventBus.emit("execution_cached", msg.data.prompt_id, msg.data.nodes); - break; - case "execution_interrupted": - this.eventBus.emit("execution_interrupted", msg.data); - break; - case "execution_error": - this.eventBus.emit("execution_error", msg.data); - break; - default: - console.warn("Unhandled message:", event.data); + if (event.data instanceof ArrayBuffer) { + const view = new DataView(event.data); + const eventType = view.getUint32(0); + const buffer = event.data.slice(4); + switch (eventType) { + case 1: + const view2 = new DataView(event.data); + const imageType = view2.getUint32(0) + let imageMime: string + switch (imageType) { + case 1: + default: + imageMime = "image/jpeg"; + break; + case 2: + imageMime = "image/png" + } + const imageBlob = new Blob([buffer.slice(4)], { type: imageMime }); + this.eventBus.emit("b_preview", imageBlob); + break; + default: + throw new Error(`Unknown binary websocket message of type ${eventType}`); + } + } + else { + const msg = JSON.parse(event.data); + switch (msg.type) { + case "status": + if (msg.data.sid) { + this.clientId = msg.data.sid; + sessionStorage["Comfy.SessionId"] = this.clientId; + } + this.eventBus.emit("status", { execInfo: { queueRemaining: msg.data.status.exec_info.queue_remaining } }); + break; + case "progress": + this.eventBus.emit("progress", msg.data as Progress); + break; + case "executing": + this.eventBus.emit("executing", msg.data.prompt_id, msg.data.node); + break; + case "executed": + this.eventBus.emit("executed", msg.data.prompt_id, msg.data.node, msg.data.output); + break; + case "execution_start": + this.eventBus.emit("execution_start", msg.data.prompt_id); + break; + case "execution_cached": + this.eventBus.emit("execution_cached", msg.data.prompt_id, msg.data.nodes); + break; + case "execution_interrupted": + this.eventBus.emit("execution_interrupted", msg.data); + break; + case "execution_error": + this.eventBus.emit("execution_error", msg.data); + break; + default: + console.warn("Unhandled message:", event.data); + } } } catch (error) { console.error("Error handling message", event.data, error); diff --git a/src/lib/components/ComfyApp.ts b/src/lib/components/ComfyApp.ts index 97abc08..0f0369c 100644 --- a/src/lib/components/ComfyApp.ts +++ b/src/lib/components/ComfyApp.ts @@ -651,6 +651,10 @@ export default class ComfyApp { } }); + this.api.addEventListener("b_preview", (imageBlob: Blob) => { + queueState.previewUpdated(imageBlob); + }); + const config = get(configState); if (config.pollSystemStatsInterval > 0) { diff --git a/src/lib/components/ComfyBoxWorkflowsView.svelte b/src/lib/components/ComfyBoxWorkflowsView.svelte index 4fd5430..722a1dd 100644 --- a/src/lib/components/ComfyBoxWorkflowsView.svelte +++ b/src/lib/components/ComfyBoxWorkflowsView.svelte @@ -386,6 +386,9 @@ + + + showLightbox(entry.images, i, e)} - src={image} + src={imageURL} + loading="lazy" alt="thumbnail" /> {/each} diff --git a/src/lib/components/PromptDisplay.svelte b/src/lib/components/PromptDisplay.svelte index b96e2c7..ee9de16 100644 --- a/src/lib/components/PromptDisplay.svelte +++ b/src/lib/components/PromptDisplay.svelte @@ -8,7 +8,7 @@ import Gallery from "$lib/components/gradio/gallery/Gallery.svelte"; import { ImageViewer } from "$lib/ImageViewer"; import type { Styles } from "@gradio/utils"; - import { comfyFileToComfyBoxMetadata, comfyURLToComfyFile, countNewLines } from "$lib/utils"; + import { comfyFileToComfyBoxMetadata, comfyURLToComfyFile, countNewLines, type ComfyImageLocation, convertComfyOutputToComfyURL } from "$lib/utils"; import ReceiveOutputTargets from "./modal/ReceiveOutputTargets.svelte"; import workflowState, { type ComfyBoxWorkflow, type WorkflowReceiveOutputTargets } from "$lib/stores/workflowState"; import type { ComfyReceiveOutputNode } from "$lib/nodes/actions"; @@ -17,7 +17,7 @@ const splitLength = 50; export let prompt: SerializedPromptInputsAll; - export let images: string[] = []; // list of image URLs to ComfyUI's /view? endpoint + export let images: ComfyImageLocation[] = []; export let isMobile: boolean = false; export let expandAll: boolean = false; export let closeModal: () => void; @@ -36,10 +36,7 @@ let litegraphType = "(none)" $: if (images.length > 0) { - // since the image links come from gradio, have to parse the URL for the - // ComfyImageLocation params - comfyBoxImages = images.map(comfyURLToComfyFile) - .map(comfyFileToComfyBoxMetadata); + comfyBoxImages = images.map(comfyFileToComfyBoxMetadata); } else { comfyBoxImages = [] @@ -199,7 +196,7 @@
= { } }; +export enum OutputThumbnailsMode { + Auto, + AlwaysThumbnail, + AlwaysFullSize +} + +const defOutputThumbnails: ConfigDefEnum<"outputThumbnails", OutputThumbnailsMode> = { + name: "outputThumbnails", + type: "enum", + defaultValue: OutputThumbnailsMode.Auto, + category: "ui", + description: "If enabled, send back smaller sized output image thumbnails for gallery/queue/history. Enable if you have slow network or are using Colab.", + options: { + values: [ + { + value: OutputThumbnailsMode.Auto, + label: "Autodetect" + }, + { + value: OutputThumbnailsMode.AlwaysThumbnail, + label: "Always use thumbnails" + }, + { + value: OutputThumbnailsMode.AlwaysFullSize, + label: "Always use full size" + }, + ] + } +}; + const defAlwaysStripUserState: ConfigDefBoolean<"alwaysStripUserState"> = { name: "alwaysStripUserState", type: "boolean", @@ -207,6 +237,7 @@ export const CONFIG_DEFS = [ defComfyUIHostname, defComfyUIPort, defNotifications, + defOutputThumbnails, defAlwaysStripUserState, defPromptForWorkflowName, defConfirmWhenUnloadingUnsavedChanges, diff --git a/src/lib/stores/uiQueueState.ts b/src/lib/stores/uiQueueState.ts index a361c22..d16af8a 100644 --- a/src/lib/stores/uiQueueState.ts +++ b/src/lib/stores/uiQueueState.ts @@ -1,7 +1,8 @@ import type { PromptID, QueueItemType } from '$lib/api'; +import type { ComfyImageLocation } from "$lib/utils"; import { get, writable } from 'svelte/store'; import type { Readable, Writable } from 'svelte/store'; -import queueState, { type CompletedQueueEntry, type QueueEntry } from './queueState'; +import queueState, { QueueEntryStatus, type CompletedQueueEntry, type QueueEntry } from './queueState'; import type { WorkflowError } from './workflowState'; import { convertComfyOutputToComfyURL } from '$lib/utils'; @@ -13,7 +14,7 @@ export type QueueUIEntry = { submessage: string, date?: string, status: QueueUIEntryStatus, - images?: string[], // URLs + images?: ComfyImageLocation[], // URLs details?: string, // shown in a tooltip on hover error?: WorkflowError } @@ -94,13 +95,12 @@ function convertPendingEntry(entry: QueueEntry, status: QueueUIEntryStatus): Que const thumbnails = entry.extraData?.thumbnails if (thumbnails) { - result.images = thumbnails.map(convertComfyOutputToComfyURL); + result.images = [...thumbnails] } const outputs = Object.values(entry.outputs) .filter(o => o.images) .flatMap(o => o.images) - .map(convertComfyOutputToComfyURL); if (outputs) { result.images = result.images.concat(outputs) } @@ -114,7 +114,6 @@ function convertCompletedEntry(entry: CompletedQueueEntry): QueueUIEntry { const images = Object.values(entry.entry.outputs) .filter(o => o.images) .flatMap(o => o.images) - .map(convertComfyOutputToComfyURL); result.images = images if (entry.message) @@ -132,6 +131,8 @@ function updateFromQueue(queuePending: QueueEntry[], queueRunning: QueueEntry[]) // newest entries appear at the top s.queuedEntries = queuePending.map((e) => convertPendingEntry(e, "pending")).reverse(); s.runningEntries = queueRunning.map((e) => convertPendingEntry(e, "running")).reverse(); + s.queuedEntries.sort((a, b) => a.entry.number - b.entry.number) + s.runningEntries.sort((a, b) => a.entry.number - b.entry.number) s.queueUIEntries = s.queuedEntries.concat(s.runningEntries); console.warn("[ComfyQueue] BUILDQUEUE", s.queuedEntries.length, s.runningEntries.length) return s; diff --git a/src/lib/utils.ts b/src/lib/utils.ts index 68bdff3..7b1df82 100644 --- a/src/lib/utils.ts +++ b/src/lib/utils.ts @@ -9,6 +9,7 @@ import workflowState, { type WorkflowReceiveOutputTargets } from "./stores/workf import { ImageViewer } from "./ImageViewer"; import configState from "$lib/stores/configState"; import SendOutputModal, { type SendOutputModalResult } from "$lib/components/modal/SendOutputModal.svelte"; +import { OutputThumbnailsMode } from "./stores/configDefs"; export function clamp(n: number, min: number, max: number): number { if (max <= min) @@ -300,31 +301,80 @@ export function convertComfyOutputToGradio(output: SerializedPromptOutput): Grad export function convertComfyOutputEntryToGradio(r: ComfyImageLocation): GradioFileData { const url = configState.getBackendURL(); - const params = new URLSearchParams(r) const fileData: GradioFileData = { name: r.filename, orig_name: r.filename, is_file: false, - data: url + "/view?" + params + data: convertComfyOutputToComfyURL(r) } return fileData } -export function convertComfyOutputToComfyURL(output: string | ComfyImageLocation): string { +function convertComfyPreviewTypeToString(preview: ComfyImagePreviewType): string { + const arr = [] + switch (preview.format) { + case ComfyImagePreviewFormat.JPEG: + arr.push("jpeg") + break; + case ComfyImagePreviewFormat.WebP: + default: + arr.push("webp") + break; + } + + arr.push(String(preview.quality)) + + return arr.join(";") +} + +export function convertComfyOutputToComfyURL(output: string | ComfyImageLocation, thumbnail: boolean = false): string { if (typeof output === "string") return output; - const params = new URLSearchParams(output) + const paramsObj = { + filename: output.filename, + subfolder: output.subfolder, + type: output.type + } + + if (thumbnail) { + let doThumbnail: boolean; + + switch (get(configState).outputThumbnails) { + case OutputThumbnailsMode.AlwaysFullSize: + doThumbnail = false; + break; + case OutputThumbnailsMode.AlwaysThumbnail: + doThumbnail = true; + break; + case OutputThumbnailsMode.Auto: + default: + // TODO detect colab, etc. + if (isMobileBrowser(navigator.userAgent)) { + doThumbnail = true; + } + else { + doThumbnail = false; + } + break; + } + + if (doThumbnail) { + output.preview = { + format: ComfyImagePreviewFormat.WebP, + quality: 80 + } + } + } + + if (output.preview != null) + paramsObj["preview"] = convertComfyPreviewTypeToString(output.preview) + + const params = new URLSearchParams(paramsObj) const url = configState.getBackendURL(); return url + "/view?" + params } -export function convertGradioFileDataToComfyURL(image: GradioFileData, type: ComfyUploadImageType = "input"): string { - const baseUrl = configState.getBackendURL(); - const params = new URLSearchParams({ filename: image.name, subfolder: "", type }) - return `${baseUrl}/view?${params}` -} - export function convertGradioFileDataToComfyOutput(fileData: GradioFileData, type: ComfyUploadImageType = "input"): ComfyImageLocation { if (!fileData.is_file) throw "Can't convert blob data to comfy output!" @@ -336,18 +386,6 @@ export function convertGradioFileDataToComfyOutput(fileData: GradioFileData, typ } } -export function convertFilenameToComfyURL(filename: string, - subfolder: string = "", - type: "input" | "output" | "temp" = "output"): string { - const params = new URLSearchParams({ - filename, - subfolder, - type - }) - const url = configState.getBackendURL(); - return url + "/view?" + params -} - export function jsonToJsObject(json: string): string { // Try to parse, to see if it's real JSON JSON.parse(json); @@ -413,6 +451,16 @@ export interface SerializedPromptOutput { [key: string]: any } +export enum ComfyImagePreviewFormat { + WebP = "webp", + JPEG = "jpeg", +} + +export type ComfyImagePreviewType = { + format: ComfyImagePreviewFormat, + quality: number +} + /** Raw output entry as received from ComfyUI's backend */ export type ComfyImageLocation = { /* Filename with extension in the subfolder. */ @@ -420,7 +468,19 @@ export type ComfyImageLocation = { /* Subfolder in the containing folder. */ subfolder: string, /* Base ComfyUI folder where the image is located. */ - type: ComfyUploadImageType + type: ComfyUploadImageType, + /* + * Preview information + * + * "format;quality" + * + * ex) + * webp;50 -> webp, quality 50 + * webp;50 -> webp, quality 50 + * jpeg;80 -> rgb, jpeg, quality 80 + * + */ + preview?: ComfyImagePreviewType } /* @@ -544,27 +604,54 @@ export function comfyBoxImageToComfyURL(image: ComfyBoxImageMetadata): string { return convertComfyOutputToComfyURL(image.comfyUIFile) } +function parseComfyUIPreviewType(previewStr: string): ComfyImagePreviewType { + let split = previewStr.split(";") + let format = ComfyImagePreviewFormat.WebP; + if (split[0] === "webp") + format = ComfyImagePreviewFormat.WebP; + else if (split[0] === "jpeg") + format = ComfyImagePreviewFormat.JPEG; + + let quality = parseInt(split[0]) + if (isNaN(quality)) + quality = 80 + + return { format, quality } +} + export function comfyURLToComfyFile(urlString: string): ComfyImageLocation | null { const url = new URL(urlString); const params = new URLSearchParams(url.search); const filename = params.get("filename") const type = params.get("type") as ComfyUploadImageType; const subfolder = params.get("subfolder") || "" + const previewStr = params.get("preview") || null; + let preview = null + + if (previewStr != null) { + preview = parseComfyUIPreviewType(preview); + } // If at least filename and type exist then we're good if (filename != null && type != null) { - return { filename, type, subfolder } + return { filename, type, subfolder, preview } } return null; } -export function showLightbox(images: string[], index: number, e: Event) { +export function showLightbox(images: ComfyImageLocation[] | string[], index: number, e: Event) { e.preventDefault() if (!images) return - ImageViewer.instance.showModal(images, index); + let images_: string[] + if (typeof images[0] === "object") + images_ = (images as ComfyImageLocation[]).map(v => convertComfyOutputToComfyURL(v)) + else + images_ = (images as string[]) + + ImageViewer.instance.showModal(images_, index); e.stopPropagation() } diff --git a/src/mobile/routes/gallery.svelte b/src/mobile/routes/gallery.svelte index 9efe72a..5c40a5a 100644 --- a/src/mobile/routes/gallery.svelte +++ b/src/mobile/routes/gallery.svelte @@ -1,17 +1,11 @@ From b126327ec2971b8b3a34442a398bc8982628c208 Mon Sep 17 00:00:00 2001 From: space-nuko <24979496+space-nuko@users.noreply.github.com> Date: Mon, 5 Jun 2023 21:24:24 -0500 Subject: [PATCH 3/3] Tag default workflow for previews use Galleries should be tagged correctly to receive previews --- public/workflows/defaultWorkflow.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/public/workflows/defaultWorkflow.json b/public/workflows/defaultWorkflow.json index 7890270..d54daac 100644 --- a/public/workflows/defaultWorkflow.json +++ b/public/workflows/defaultWorkflow.json @@ -61,7 +61,7 @@ ], "title": "UI.Gallery", "properties": { - "tags": [], + "tags": ["gen"], "defaultValue": [], "index": 3, "updateMode": "append", @@ -1694,7 +1694,7 @@ ], "title": "UI.Gallery", "properties": { - "tags": [], + "tags": ["hr"], "defaultValue": [], "index": 1, "updateMode": "append",