From 9a40f84e79aaeb5c4d9ff45420b2c6fe7960fd47 Mon Sep 17 00:00:00 2001 From: space-nuko <24979496+space-nuko@users.noreply.github.com> Date: Tue, 23 May 2023 12:43:13 -0500 Subject: [PATCH] Region widget --- src/lib/nodes/widgets/ComfyMultiRegionNode.ts | 55 +++-- src/lib/widgets/MultiRegionWidget.svelte | 190 ++++++++++++------ 2 files changed, 164 insertions(+), 81 deletions(-) diff --git a/src/lib/nodes/widgets/ComfyMultiRegionNode.ts b/src/lib/nodes/widgets/ComfyMultiRegionNode.ts index 94de3af..5c2b934 100644 --- a/src/lib/nodes/widgets/ComfyMultiRegionNode.ts +++ b/src/lib/nodes/widgets/ComfyMultiRegionNode.ts @@ -14,18 +14,20 @@ function isBoundingBox(param: any): param is BoundingBox { export interface ComfyMultiRegionProperties extends ComfyWidgetProperties { regionCount: number, - totalWidth: number, - totalHeight: number, + canvasWidth: number, + canvasHeight: number, inputType: "size" | "image" } +const DEFAULT_BBOX: BoundingBox = [0.4, 0.4, 0.2, 0.2]; + export default class ComfyMultiRegionNode extends ComfyWidgetNode { override properties: ComfyMultiRegionProperties = { tags: [], defaultValue: false, regionCount: 1, - totalWidth: 512, - totalHeight: 512, + canvasWidth: 512, + canvasHeight: 512, inputType: "size" } @@ -49,12 +51,13 @@ export default class ComfyMultiRegionNode extends ComfyWidgetNode } override svelteComponentType = MultiRegionWidget; - override defaultValue: BoundingBox[] = [[0.4, 0.4, 0.8, 0.2]]; + override defaultValue: BoundingBox[] = [[...DEFAULT_BBOX]]; override outputSlotName = null; override storeActionName = "store"; override changedEventName = "changed"; sizeChanged: Writable = writable(true); + regionsChanged: Writable = writable(true); override onPropertyChanged(property: any, value: any) { if (property === "regionCount") { @@ -66,16 +69,18 @@ export default class ComfyMultiRegionNode extends ComfyWidgetNode } constructor(name?: string) { - super(name, [[0.4, 0.4, 0.8, 0.2]]) + super(name, [[...DEFAULT_BBOX]]) } override onExecute() { - let width = this.getInputData(1) - let height = this.getInputData(2) + let width = this.getInputData(1) || 0 + let height = this.getInputData(2) || 0 - if (width != null && height != null && width != this.properties.width && height != this.properties.height) { - this.properties.width = width; - this.properties.height = height; + if (width != this.properties.canvasWidth || height != this.properties.canvasHeight) { + console.warn("SIZCHANGE", width, height, this.properties.canvasWidth, this.properties.canvasHeight) + this.properties.canvasWidth = width; + this.properties.canvasHeight = height; + this.sizeChanged.set(true); this.updateSize(); } @@ -86,10 +91,10 @@ export default class ComfyMultiRegionNode extends ComfyWidgetNode if (bbox != null) { const xOutput = this.outputs[index + 1] if (xOutput != null) { - this.setOutputData(index + 1, bbox[0] * this.properties.width) - this.setOutputData(index + 2, bbox[1] * this.properties.height) - this.setOutputData(index + 3, bbox[2] * this.properties.width) - this.setOutputData(index + 4, bbox[3] * this.properties.height) + this.setOutputData(index + 1, bbox[0] * this.properties.canvasWidth) + this.setOutputData(index + 2, bbox[1] * this.properties.canvasHeight) + this.setOutputData(index + 3, bbox[2] * this.properties.canvasWidth) + this.setOutputData(index + 4, bbox[3] * this.properties.canvasHeight) } } } @@ -111,12 +116,15 @@ export default class ComfyMultiRegionNode extends ComfyWidgetNode this.addOutput(`h${index + 1}`, "number") } + this.regionsChanged.set(true); + this.notifyPropsChanged(); + this.setValue(this.getValue()) } private updateSize(value?: BoundingBox[]): BoundingBox[] { - this.properties.width = Math.max(this.properties.width, 1); - this.properties.height = Math.max(this.properties.height, 1); + this.properties.canvasWidth = Math.max(this.properties.canvasWidth, 0); + this.properties.canvasHeight = Math.max(this.properties.canvasHeight, 0); value ||= this.getValue(); @@ -128,6 +136,7 @@ export default class ComfyMultiRegionNode extends ComfyWidgetNode } this.sizeChanged.set(true); + this.notifyPropsChanged(); return value } @@ -136,13 +145,19 @@ export default class ComfyMultiRegionNode extends ComfyWidgetNode if (param == null || this.properties.regionCount <= 0) return [] + let val = [] + if (isBoundingBox(param)) - return this.updateSize([param]) + val = this.updateSize([param]) if (Array.isArray(param) && param.every(isBoundingBox)) - return this.updateSize(param.splice(0, this.properties.regionCount)) + val = this.updateSize(param.splice(0, this.properties.regionCount)) - return null; + // Fill the array with missing regions + for (let index = val.length; index < this.properties.regionCount; index++) + val.push([...DEFAULT_BBOX]) + + return val; } } diff --git a/src/lib/widgets/MultiRegionWidget.svelte b/src/lib/widgets/MultiRegionWidget.svelte index 179e073..26cb182 100644 --- a/src/lib/widgets/MultiRegionWidget.svelte +++ b/src/lib/widgets/MultiRegionWidget.svelte @@ -8,28 +8,30 @@ import type { ComfyMultiRegionNode } from "$lib/nodes/widgets"; import type { BoundingBox } from "$lib/nodes/widgets/ComfyMultiRegionNode"; import type { WidgetLayout } from "$lib/stores/layoutStates"; - import { Block } from "@gradio/atoms"; + import { Block, BlockLabel } from "@gradio/atoms"; + import { Chart as SquareIcon } from "@gradio/icons"; import { writable, type Writable } from "svelte/store"; import { generateBlankCanvas, loadImage } from "./utils"; + import { clamp } from "$lib/utils"; // ref: https://html-color.codes/ const COLOR_MAP: [string, string][] = [ - ['#ff0000', '2px solid rgba(255, 0, 0, 0.3)'], // red - ['#ff9900', '2px solid rgba(255, 153, 0, 0.3)'], // orange - ['#ffff00', '2px solid rgba(255, 255, 0, 0.3)'], // yellow - ['#33cc33', '2px solid rgba(51, 204, 51, 0.3)'], // green - ['#33cccc', '2px solid rgba(51, 204, 204, 0.3)'], // indigo - ['#0066ff', '2px solid rgba(0, 102, 255, 0.3)'], // blue - ['#6600ff', '2px solid rgba(102, 0, 255, 0.3)'], // purple - ['#cc00cc', '2px solid rgba(204, 0, 204, 0.3)'], // dark pink - ['#ff6666', '2px solid rgba(255, 102, 102, 0.3)'], // light red - ['#ffcc66', '2px solid rgba(255, 204, 102, 0.3)'], // light orange - ['#99cc00', '2px solid rgba(153, 204, 0, 0.3)'], // lime green - ['#00cc99', '2px solid rgba(0, 204, 153, 0.3)'], // teal - ['#0099cc', '2px solid rgba(0, 153, 204, 0.3)'], // steel blue - ['#9933cc', '2px solid rgba(153, 51, 204, 0.3)'], // lavender - ['#ff3399', '2px solid rgba(255, 51, 153, 0.3)'], // hot pink - ['#996633', '2px solid rgba(153, 102, 51, 0.3)'], // brown + ['#ff0000', 'rgba(255, 0, 0, 0.3)'], // red + ['#ff9900', 'rgba(255, 153, 0, 0.3)'], // orange + ['#ffff00', 'rgba(255, 255, 0, 0.3)'], // yellow + ['#33cc33', 'rgba(51, 204, 51, 0.3)'], // green + ['#33cccc', 'rgba(51, 204, 204, 0.3)'], // indigo + ['#0066ff', 'rgba(0, 102, 255, 0.3)'], // blue + ['#6600ff', 'rgba(102, 0, 255, 0.3)'], // purple + ['#cc00cc', 'rgba(204, 0, 204, 0.3)'], // dark pink + ['#ff6666', 'rgba(255, 102, 102, 0.3)'], // light red + ['#ffcc66', 'rgba(255, 204, 102, 0.3)'], // light orange + ['#99cc00', 'rgba(153, 204, 0, 0.3)'], // lime green + ['#00cc99', 'rgba(0, 204, 153, 0.3)'], // teal + ['#0099cc', 'rgba(0, 153, 204, 0.3)'], // steel blue + ['#9933cc', 'rgba(153, 51, 204, 0.3)'], // lavender + ['#ff3399', 'rgba(255, 51, 153, 0.3)'], // hot pink + ['#996633', 'rgba(153, 102, 51, 0.3)'], // brown ]; export let widget: WidgetLayout | null = null; @@ -41,17 +43,29 @@ const COLOR_MAP: [string, string][] = [ let node: ComfyMultiRegionNode | null = null; let nodeValue: Writable = writable([]); let sizeChanged: Writable = writable(false); + let regionsChanged: Writable = writable(false); + let propsChanged: Writable = writable(0); + let selectedIndex: number = 0; $: widget && setNodeValue(widget); function setNodeValue(widget: WidgetLayout) { + console.error("SETNODEVALUE") if (widget) { node = widget.node as ComfyMultiRegionNode nodeValue = node.value; + propsChanged = node.propsChanged; sizeChanged = node.sizeChanged; + regionsChanged = node.regionsChanged; } }; + let showWidget: boolean = false; + + if (sizeChanged && $sizeChanged) { + console.error("Z@"); + } + type DisplayBoundingBox = { xPx: number, yPx: number, @@ -62,8 +76,7 @@ const COLOR_MAP: [string, string][] = [ borderColor: string } - let displayBoxes = [] - let changed = true; + let displayBoxes = []; async function recreateDisplayBoxes(_node?: ComfyMultiRegionNode, bboxes?: BoundingBox[]): Promise { _node ||= node; @@ -72,7 +85,8 @@ const COLOR_MAP: [string, string][] = [ console.debug("[MultiRegionWidget] Recreate!", bboxes, imageElem, _node) if (_node != null && imageElem != null && imageContainer != null) { - await updateImage(_node.properties.totalWidth, _node.properties.totalHeight); + selectedIndex = clamp(selectedIndex, 0, bboxes.length - 1); + await updateImage(_node.properties.canvasWidth, _node.properties.canvasHeight); return bboxes.map((b, i) => displayBoundingBox(b, i, imageElem)) } else { @@ -81,21 +95,29 @@ const COLOR_MAP: [string, string][] = [ } $: if (node != null && $sizeChanged) { - updateImage(node.properties.totalWidth, node.properties.totalHeight) - .then(() => $sizeChanged = false) + console.warn("SIZCHANGEd") + updateImage(node.properties.canvasWidth, node.properties.canvasHeight) + .then(() => { + return recreateDisplayBoxes() + }) + .then(dbs => { + displayBoxes = dbs; + }) } onMount(async () => { displayBoxes = await recreateDisplayBoxes(node, $nodeValue); }) - $: if (changed) { - changed = false; + $: if ($regionsChanged) { + $regionsChanged = false; recreateDisplayBoxes(node, $nodeValue).then(dbs => displayBoxes = dbs); } async function updateImage(width: number, height: number) { - const blank = generateBlankCanvas(width, height); + showWidget = width > 0 && height > 0; + console.error("SHOW", showWidget, width, height) + const blank = generateBlankCanvas(width, height, "transparent"); const url = blank.toDataURL(); const newImg = await loadImage(url); newImg.classList.add("regions-image"); @@ -103,6 +125,7 @@ const COLOR_MAP: [string, string][] = [ imageContainer.replaceChildren(newImg) } imageElem = newImg; + imageElem.style.border = `${BORDER_SIZE_PX}px solid var(--border-color-primary)`; $sizeChanged = false; } @@ -119,14 +142,14 @@ const COLOR_MAP: [string, string][] = [ // client: image widget display size // natural: content image real size const vpScale = Math.min(imageElem.clientWidth / imageElem.naturalWidth, imageElem.clientHeight / imageElem.naturalHeight); - const imageElemCenterX = imageElem.clientWidth / 2; - const imageElemCenterY = imageElem.clientHeight / 2; + const imageElemCenterX = (imageElem.clientWidth) / 2; + const imageElemCenterY = (imageElem.clientHeight) / 2; const scaledX = imageElem.naturalWidth * vpScale; const scaledY = imageElem.naturalHeight * vpScale; - const viewRectLeft = imageElemCenterX - scaledX / 2; - const viewRectRight = imageElemCenterX + scaledX / 2; - const viewRectTop = imageElemCenterY - scaledY / 2; - const viewRectDown = imageElemCenterY + scaledY / 2; + const viewRectLeft = imageElemCenterX - scaledX / 2 + BORDER_SIZE_PX; + const viewRectRight = imageElemCenterX + scaledX / 2 + BORDER_SIZE_PX; + const viewRectTop = imageElemCenterY - scaledY / 2 + BORDER_SIZE_PX; + const viewRectDown = imageElemCenterY + scaledY / 2 + BORDER_SIZE_PX; const xDiv = viewRectLeft + scaledX * x; const yDiv = viewRectTop + scaledY * y; @@ -143,7 +166,7 @@ const COLOR_MAP: [string, string][] = [ const maxH = maxSize / scaledY; const warnLargeSize = w > maxW || h > maxH - const [bgColor, borderColor] = COLOR_MAP[index % COLOR_MAP.length] + const [borderColor, bgColor] = COLOR_MAP[index % COLOR_MAP.length] out ||= {} as DisplayBoundingBox out.xPx= xDiv; @@ -159,6 +182,7 @@ const COLOR_MAP: [string, string][] = [ const RESIZE_BORDER = 5; const MOVE_BORDER = 5; + const BORDER_SIZE_PX = 3; function updateCursorStyle(e: MouseEvent) { // This function changes the cursor style when hovering over the bounding box @@ -194,6 +218,8 @@ const COLOR_MAP: [string, string][] = [ if (!imageElem || !bbox) return; + selectedIndex = index; + // Check if the click is inside the bounding box const div = e.target as HTMLDivElement; const boxRect = div.getBoundingClientRect(); @@ -344,7 +370,7 @@ const COLOR_MAP: [string, string][] = [ const onMouseUp = () => { document.removeEventListener('mousemove', onMouseMove); document.removeEventListener('mouseup', onMouseUp); - changed = true; + $regionsChanged = true; $nodeValue = $nodeValue; } @@ -360,38 +386,65 @@ const COLOR_MAP: [string, string][] = [ - -
-
- -
-
- {#each displayBoxes as dBox, i} -
onBoxMouseDown(e, i)} - > - - Warning: Region very large! - +{#key $propsChanged} + + {#if widget?.attrs.title} + {@const label = widget.attrs.title} + + {/if} + {#if showWidget} +
+
+
- {/each} -
-
- +
+ {#each displayBoxes as dBox, i} + {@const selected = selectedIndex === i} +
onBoxMouseDown(e, i)} + > + + Warning: Region very large! + +
+ {/each} +
+
+ {:else} +
+ (Empty canvas) +
+ {/if} + +{/key}