neuro_llc/ComfyBox
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 1 Wiki Activity

Region widget WIP

Browse Source
This commit is contained in:
space-nuko
2023-05-23 11:48:46 -05:00
parent 88b9477402
commit 6ebd8c75a1
6 changed files with 617 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
README.md
View File

@@ -26,7 +26,7 @@ Also note that the saved workflow format is subject to change until it's been fi
## Features
- **No-Code UI Builder** - A novel system for creating your own Stable Diffusion user interfaces from the basic components.
- **Manage Multiple Workflows** - You can open as many workflows as you like and switch between them using tabs within the app.
- **Manage Multiple Workflows** - You can open as many workflows as you like and switch between them using tabs within the app. You can also write custom "Send To" actions to copy your image outputs into other workflows for further processing.
- **Use Your Existing Workflows** - Import workflows you've created in ComfyUI into ComfyBox and a new UI will be created for you.
- **Extension Support** - All custom ComfyUI nodes are supported out of the box.
- **Prompt Queue** - Queue up multiple prompts without waiting for them to finish first. Inspect currently queued and executed prompts.

154
src/lib/nodes/widgets/ComfyMultiRegionNode.ts Normal file
View File

@@ -0,0 +1,154 @@
import { BuiltInSlotType, LiteGraph, type SlotLayout } from "@litegraph-ts/core";
import MultiRegionWidget from "$lib/widgets/MultiRegionWidget.svelte";
import ComfyWidgetNode, { type ComfyWidgetProperties } from "./ComfyWidgetNode";
import { clamp } from "$lib/utils";
import { writable, type Writable } from "svelte/store";
/* x, y, width, height, all in range 0.0 - 1.0 */
export type BoundingBox = [number, number, number, number]
function isBoundingBox(param: any): param is BoundingBox {
return Array.isArray(param) && param.length === 4 && param.every(i => typeof i === "number")
}
export interface ComfyMultiRegionProperties extends ComfyWidgetProperties {
regionCount: number,
totalWidth: number,
totalHeight: number,
inputType: "size" | "image"
}
export default class ComfyMultiRegionNode extends ComfyWidgetNode<BoundingBox[]> {
override properties: ComfyMultiRegionProperties = {
tags: [],
defaultValue: false,
regionCount: 1,
totalWidth: 512,
totalHeight: 512,
inputType: "size"
}
static slotLayout: SlotLayout = {
inputs: [
{ name: "store", type: BuiltInSlotType.ACTION },
// dynamic inputs, may be removed later
{ name: "width", type: "number" },
{ name: "height", type: "number" },
],
outputs: [
{ name: "changed", type: BuiltInSlotType.EVENT },
// dynamic outputs, may be removed later
{ name: "x1", type: "number" },
{ name: "y1", type: "number" },
{ name: "w1", type: "number" },
{ name: "h1", type: "number" },
]
}
override svelteComponentType = MultiRegionWidget;
override defaultValue: BoundingBox[] = [[0.4, 0.4, 0.8, 0.2]];
override outputSlotName = null;
override storeActionName = "store";
override changedEventName = "changed";
sizeChanged: Writable<boolean> = writable(true);
override onPropertyChanged(property: any, value: any) {
if (property === "regionCount") {
this.updateRegions()
}
else if (property === "width" || property === "height") {
this.updateSize();
}
}
constructor(name?: string) {
super(name, [[0.4, 0.4, 0.8, 0.2]])
}
override onExecute() {
let width = this.getInputData(1)
let height = this.getInputData(2)
if (width != null && height != null && width != this.properties.width && height != this.properties.height) {
this.properties.width = width;
this.properties.height = height;
this.updateSize();
}
const value = this.getValue();
for (let index = 0; index < this.properties.regionCount * 2; index += 2) {
const bbox = value[index]
if (bbox != null) {
const xOutput = this.outputs[index + 1]
if (xOutput != null) {
this.setOutputData(index + 1, bbox[0] * this.properties.width)
this.setOutputData(index + 2, bbox[1] * this.properties.height)
this.setOutputData(index + 3, bbox[2] * this.properties.width)
this.setOutputData(index + 4, bbox[3] * this.properties.height)
}
}
}
}
private updateRegions() {
this.properties.regionCount = Math.max(this.properties.regionCount, 0);
for (let index = this.outputs.length - 1; index >= 0; index--) {
if (this.outputs[index].type !== BuiltInSlotType.EVENT) {
this.removeOutput(index);
}
}
for (let index = 0; index < this.properties.regionCount; index++) {
this.addOutput(`x${index + 1}`, "number")
this.addOutput(`y${index + 1}`, "number")
this.addOutput(`w${index + 1}`, "number")
this.addOutput(`h${index + 1}`, "number")
}
this.setValue(this.getValue())
}
private updateSize(value?: BoundingBox[]): BoundingBox[] {
this.properties.width = Math.max(this.properties.width, 1);
this.properties.height = Math.max(this.properties.height, 1);
value ||= this.getValue();
for (const bbox of value) {
bbox[0] = clamp(bbox[0], 0, 1 - bbox[2]);
bbox[1] = clamp(bbox[1], 0, 1 - bbox[3]);
bbox[2] = clamp(bbox[2], 0, 1 - bbox[1])
bbox[3] = clamp(bbox[3], 0, 1 - bbox[2])
}
this.sizeChanged.set(true);
return value
}
override parseValue(param: any): BoundingBox[] {
if (param == null || this.properties.regionCount <= 0)
return []
if (isBoundingBox(param))
return this.updateSize([param])
if (Array.isArray(param) && param.every(isBoundingBox))
return this.updateSize(param.splice(0, this.properties.regionCount))
return null;
}
}
LiteGraph.registerNodeType({
class: ComfyMultiRegionNode,
title: "UI.MultiRegion",
desc: "Overlays one or more regions over a canvas of the given width/height",
type: "ui/multi_region"
})

1
src/lib/nodes/widgets/index.ts
View File

@@ -8,3 +8,4 @@ export { default as ComfyImageUploadNode } from "./ComfyImageUploadNode"
export { default as ComfyRadioNode } from "./ComfyRadioNode"
export { default as ComfyNumberNode } from "./ComfyNumberNode"
export { default as ComfyTextNode } from "./ComfyTextNode"
export { default as ComfyMultiRegionNode } from "./ComfyMultiRegionNode"

37
src/lib/widgets/ImageUploadWidget.svelte
View File

@@ -15,6 +15,7 @@
import NumberInput from "$lib/components/NumberInput.svelte";
import type { ComfyImageEditorNode } from "$lib/nodes/widgets";
import { ImageViewer } from "$lib/ImageViewer";
import { generateBlankCanvas, generateImageCanvas } from "./utils";
export let widget: WidgetLayout | null = null;
export let isMobile: boolean = false;
@@ -55,42 +56,6 @@
showModal = false;
}
function generateBlankCanvas(width: number, height: number, fill: string = "#fff"): HTMLCanvasElement {
const canvas = document.createElement('canvas');
canvas.width = width;
canvas.height = height;
const ctx = canvas.getContext('2d');
ctx.save();
ctx.fillStyle = fill,
ctx.fillRect(0, 0, canvas.width, canvas.height);
ctx.restore();
return canvas;
}
async function loadImage(imageURL: string): Promise<HTMLImageElement> {
return new Promise((resolve) => {
const e = new Image();
e.setAttribute('crossorigin', 'anonymous'); // Don't taint the canvas from loading files on-disk
e.addEventListener("load", () => { resolve(e); });
e.src = imageURL;
return e;
});
}
async function generateImageCanvas(imageURL: string): Promise<[HTMLCanvasElement, number, number]> {
const image = await loadImage(imageURL);
const canvas = document.createElement('canvas');
canvas.width = image.width;
canvas.height = image.height;
const ctx = canvas.getContext('2d');
ctx.save();
ctx.fillStyle = "rgba(255, 255, 255, 0.0)";
ctx.fillRect(0, 0, canvas.width, canvas.height);
ctx.drawImage(image, 0, 0);
ctx.restore();
return [canvas, image.width, image.height];
}
const FILENAME: string = "ComfyUITemp.png";
const SUBFOLDER: string = "ComfyBox_Editor";
const DIRECTORY: ComfyUploadImageType = "input";

424
src/lib/widgets/MultiRegionWidget.svelte Normal file
View File

@@ -0,0 +1,424 @@
<script lang="ts">
/*
* Portions of this code were adapted from multidiffusion-upscaler-for-automatic1111 ver. 2023.3.28
* which was licensed under GPL-3. Subsequent versions were relicensed as CC BY-NC-SA 4.0
* https://github.com/pkuliyi2015/multidiffusion-upscaler-for-automatic1111/commit/e116fe946fe44c52d5500ace9ce65d4d685cff46
*/
import { onMount } from "svelte";
import type { ComfyMultiRegionNode } from "$lib/nodes/widgets";
import type { BoundingBox } from "$lib/nodes/widgets/ComfyMultiRegionNode";
import type { WidgetLayout } from "$lib/stores/layoutStates";
import { Block } from "@gradio/atoms";
import { writable, type Writable } from "svelte/store";
import { generateBlankCanvas, loadImage } from "./utils";
// ref: https://html-color.codes/
const COLOR_MAP: [string, string][] = [
['#ff0000', '2px solid rgba(255, 0, 0, 0.3)'], // red
['#ff9900', '2px solid rgba(255, 153, 0, 0.3)'], // orange
['#ffff00', '2px solid rgba(255, 255, 0, 0.3)'], // yellow
['#33cc33', '2px solid rgba(51, 204, 51, 0.3)'], // green
['#33cccc', '2px solid rgba(51, 204, 204, 0.3)'], // indigo
['#0066ff', '2px solid rgba(0, 102, 255, 0.3)'], // blue
['#6600ff', '2px solid rgba(102, 0, 255, 0.3)'], // purple
['#cc00cc', '2px solid rgba(204, 0, 204, 0.3)'], // dark pink
['#ff6666', '2px solid rgba(255, 102, 102, 0.3)'], // light red
['#ffcc66', '2px solid rgba(255, 204, 102, 0.3)'], // light orange
['#99cc00', '2px solid rgba(153, 204, 0, 0.3)'], // lime green
['#00cc99', '2px solid rgba(0, 204, 153, 0.3)'], // teal
['#0099cc', '2px solid rgba(0, 153, 204, 0.3)'], // steel blue
['#9933cc', '2px solid rgba(153, 51, 204, 0.3)'], // lavender
['#ff3399', '2px solid rgba(255, 51, 153, 0.3)'], // hot pink
['#996633', '2px solid rgba(153, 102, 51, 0.3)'], // brown
];
export let widget: WidgetLayout | null = null;
export let isMobile: boolean = false;
let imageContainer: HTMLDivElement | null;
let imageElem: HTMLImageElement | null;
let node: ComfyMultiRegionNode | null = null;
let nodeValue: Writable<BoundingBox[]> = writable([]);
let sizeChanged: Writable<boolean> = writable(false);
$: widget && setNodeValue(widget);
function setNodeValue(widget: WidgetLayout) {
if (widget) {
node = widget.node as ComfyMultiRegionNode
nodeValue = node.value;
sizeChanged = node.sizeChanged;
}
};
type DisplayBoundingBox = {
xPx: number,
yPx: number,
widthPx: number,
heightPx: number,
warnLargeSize: boolean,
bgColor: string,
borderColor: string
}
let displayBoxes = []
let changed = true;
async function recreateDisplayBoxes(_node?: ComfyMultiRegionNode, bboxes?: BoundingBox[]): Promise<DisplayBoundingBox[]> {
_node ||= node;
bboxes ||= $nodeValue
console.debug("[MultiRegionWidget] Recreate!", bboxes, imageElem, _node)
if (_node != null && imageElem != null && imageContainer != null) {
await updateImage(_node.properties.totalWidth, _node.properties.totalHeight);
return bboxes.map((b, i) => displayBoundingBox(b, i, imageElem))
}
else {
return []
}
}
$: if (node != null && $sizeChanged) {
updateImage(node.properties.totalWidth, node.properties.totalHeight)
.then(() => $sizeChanged = false)
}
onMount(async () => {
displayBoxes = await recreateDisplayBoxes(node, $nodeValue);
})
$: if (changed) {
changed = false;
recreateDisplayBoxes(node, $nodeValue).then(dbs => displayBoxes = dbs);
}
async function updateImage(width: number, height: number) {
const blank = generateBlankCanvas(width, height);
const url = blank.toDataURL();
const newImg = await loadImage(url);
newImg.classList.add("regions-image");
if (imageContainer != null) {
imageContainer.replaceChildren(newImg)
}
imageElem = newImg;
$sizeChanged = false;
}
const BBOX_WARNING_SIZE = 1280;
function onBBoxMoved(index: number, newBBox: BoundingBox) {
$nodeValue[index] = newBBox;
displayBoxes[index] = displayBoundingBox(newBBox, index, imageElem, displayBoxes[index]);
}
function displayBoundingBox(bbox: BoundingBox, index: number, imageElem: HTMLImageElement, out?: DisplayBoundingBox): DisplayBoundingBox {
const [x, y, w, h] = bbox;
// client: image widget display size
// natural: content image real size
const vpScale = Math.min(imageElem.clientWidth / imageElem.naturalWidth, imageElem.clientHeight / imageElem.naturalHeight);
const imageElemCenterX = imageElem.clientWidth / 2;
const imageElemCenterY = imageElem.clientHeight / 2;
const scaledX = imageElem.naturalWidth * vpScale;
const scaledY = imageElem.naturalHeight * vpScale;
const viewRectLeft = imageElemCenterX - scaledX / 2;
const viewRectRight = imageElemCenterX + scaledX / 2;
const viewRectTop = imageElemCenterY - scaledY / 2;
const viewRectDown = imageElemCenterY + scaledY / 2;
const xDiv = viewRectLeft + scaledX * x;
const yDiv = viewRectTop + scaledY * y;
const wDiv = Math.min(scaledX * w, viewRectRight - xDiv);
const hDiv = Math.min(scaledY * h, viewRectDown - yDiv);
// Calculate warning bbox size
const upscalerFactor = 1.0;
// if (!is_t2i) {
// upscalerFactor = getUpscalerFactor();
// }
const maxSize = BBOX_WARNING_SIZE / upscalerFactor * vpScale;
const maxW = maxSize / scaledX;
const maxH = maxSize / scaledY;
const warnLargeSize = w > maxW || h > maxH
const [bgColor, borderColor] = COLOR_MAP[index % COLOR_MAP.length]
out ||= {} as DisplayBoundingBox
out.xPx= xDiv;
out.yPx= yDiv;
out.widthPx= wDiv;
out.heightPx= hDiv;
out.warnLargeSize = warnLargeSize;
out.bgColor = bgColor
out.borderColor = borderColor
return out;
}
const RESIZE_BORDER = 5;
const MOVE_BORDER = 5;
function updateCursorStyle(e: MouseEvent) {
// This function changes the cursor style when hovering over the bounding box
const div = e.target as HTMLDivElement;
const boxRect = div.getBoundingClientRect();
const mouseX = e.clientX;
const mouseY = e.clientY;
const resizeLeft = mouseX >= boxRect.left && mouseX <= boxRect.left + RESIZE_BORDER;
const resizeRight = mouseX >= boxRect.right - RESIZE_BORDER && mouseX <= boxRect.right;
const resizeTop = mouseY >= boxRect.top && mouseY <= boxRect.top + RESIZE_BORDER;
const resizeBottom = mouseY >= boxRect.bottom - RESIZE_BORDER && mouseY <= boxRect.bottom;
if ((resizeLeft && resizeTop) || (resizeRight && resizeBottom)) {
div.style.cursor = 'nwse-resize';
} else if ((resizeLeft && resizeBottom) || (resizeRight && resizeTop)) {
div.style.cursor = 'nesw-resize';
} else if (resizeLeft || resizeRight) {
div.style.cursor = 'ew-resize';
} else if (resizeTop || resizeBottom) {
div.style.cursor = 'ns-resize';
} else {
div.style.cursor = 'move';
}
}
function onBoxMouseDown(e: MouseEvent, index: number) {
if (e.button !== 0)
return;
// Get the bounding box
let bbox = $nodeValue[index];
if (!imageElem || !bbox)
return;
// Check if the click is inside the bounding box
const div = e.target as HTMLDivElement;
const boxRect = div.getBoundingClientRect();
let mouseX = e.clientX;
let mouseY = e.clientY;
const resizeLeft = mouseX >= boxRect.left && mouseX <= boxRect.left + RESIZE_BORDER;
const resizeRight = mouseX >= boxRect.right - RESIZE_BORDER && mouseX <= boxRect.right;
const resizeTop = mouseY >= boxRect.top && mouseY <= boxRect.top + RESIZE_BORDER;
const resizeBottom = mouseY >= boxRect.bottom - RESIZE_BORDER && mouseY <= boxRect.bottom;
const moveHorizontal = mouseX >= boxRect.left + MOVE_BORDER && mouseX <= boxRect.right - MOVE_BORDER;
const moveVertical = mouseY >= boxRect.top + MOVE_BORDER && mouseY <= boxRect.bottom - MOVE_BORDER;
if (!resizeLeft && !resizeRight && !resizeTop && !resizeBottom && !moveHorizontal && !moveVertical)
return;
const horizontalPivot = resizeLeft ? bbox[0] + bbox[2] : bbox[0];
const verticalPivot = resizeTop ? bbox[1] + bbox[3] : bbox[1];
// Canvas can be regarded as invariant during the drag operation
// Calculate in advance to reduce overhead
// Calculate viewport scale based on the current canvas size and the natural image size
let vpScale = Math.min(imageElem.clientWidth / imageElem.naturalWidth, imageElem.clientHeight / imageElem.naturalHeight);
let vpOffset = imageElem.getBoundingClientRect();
console.warn(vpScale, vpOffset)
// Calculate scaled dimensions of the canvas
let scaledX = imageElem.naturalWidth * vpScale;
let scaledY = imageElem.naturalHeight * vpScale;
// Calculate the canvas center and view rectangle coordinates
let canvasCenterX = (vpOffset.left + window.scrollX) + imageElem.clientWidth / 2;
let canvasCenterY = (vpOffset.top + window.scrollY) + imageElem.clientHeight / 2;
let viewRectLeft = canvasCenterX - scaledX / 2 - window.scrollX;
let viewRectRight = canvasCenterX + scaledX / 2 - window.scrollX;
let viewRectTop = canvasCenterY - scaledY / 2 - window.scrollY;
let viewRectDown = canvasCenterY + scaledY / 2 - window.scrollY;
mouseX = Math.min(Math.max(mouseX, viewRectLeft), viewRectRight);
mouseY = Math.min(Math.max(mouseY, viewRectTop), viewRectDown);
// Move or resize the bounding box on mousemove
function onMouseMove(e) {
// Prevent selecting anything irrelevant
e.preventDefault();
// Get the new mouse position
let newMouseX = e.clientX;
let newMouseY = e.clientY;
// clamp the mouse position to the view rectangle
newMouseX = Math.min(Math.max(newMouseX, viewRectLeft), viewRectRight);
newMouseY = Math.min(Math.max(newMouseY, viewRectTop), viewRectDown);
// Calculate the mouse movement delta
const dx = (newMouseX - mouseX) / scaledX;
const dy = (newMouseY - mouseY) / scaledY;
// Update the mouse position
mouseX = newMouseX;
mouseY = newMouseY;
// if no move just return
if (dx === 0 && dy === 0) { return; }
// Update the mouse position
let [x, y, w, h] = bbox;
if (moveHorizontal && moveVertical) {
// If moving the bounding box
x = Math.min(Math.max(x + dx, 0), 1 - w);
y = Math.min(Math.max(y + dy, 0), 1 - h);
} else {
// If resizing the bounding box
if (resizeLeft || resizeRight) {
if (x < horizontalPivot) {
if (dx <= w) {
// If still within the left side of the pivot
x = x + dx;
w = w - dx;
} else {
// If crossing the pivot
w = dx - w;
x = horizontalPivot;
}
} else {
if (w + dx < 0) {
// If still within the right side of the pivot
x = horizontalPivot + w + dx;
w = - dx - w;
} else {
// If crossing the pivot
x = horizontalPivot;
w = w + dx;
}
}
// Clamp the bounding box to the image
if (x < 0) {
w = w + x;
x = 0;
} else if (x + w > 1) {
w = 1 - x;
}
}
// Same as above, but for the vertical axis
if (resizeTop || resizeBottom) {
if (y < verticalPivot) {
if (dy <= h) {
y = y + dy;
h = h - dy;
} else {
h = dy - h;
y = verticalPivot;
}
} else {
if (h + dy < 0) {
y = verticalPivot + h + dy;
h = - dy - h;
} else {
y = verticalPivot;
h = h + dy;
}
}
if (y < 0) {
h = h + y;
y = 0;
} else if (y + h > 1) {
h = 1 - y;
}
}
}
const [div, old_bbox, _] = $nodeValue[index];
// If all the values are the same, just return
if (old_bbox[0] === x && old_bbox[1] === y && old_bbox[2] === w && old_bbox[3] === h) { return; }
// else update the bbox
bbox[0] = x;
bbox[1] = y;
bbox[2] = w;
bbox[3] = h;
onBBoxMoved(index, bbox);
}
const onMouseUp = () => {
document.removeEventListener('mousemove', onMouseMove);
document.removeEventListener('mouseup', onMouseUp);
changed = true;
$nodeValue = $nodeValue;
}
// Add the event listeners
document.addEventListener('mousemove', onMouseMove);
document.addEventListener('mouseup', onMouseUp);
}
async function onResize() {
displayBoxes = await recreateDisplayBoxes();
}
</script>
<svelte:window on:resize={onResize}/>
<Block>
<div class="regions-container">
<div bind:this={imageContainer} class="regions-image-container">
<img bind:this={imageElem} class="regions-image"/>
</div>
<div class="regions">
{#each displayBoxes as dBox, i}
<div class="region"
style:left="{dBox.xPx}px"
style:top="{dBox.yPx}px"
style:width="{dBox.widthPx}px"
style:height="{dBox.heightPx}px"
style:background={dBox.bgColor}
style:border={dBox.borderColor}
style:display="block"
on:mousemove={updateCursorStyle}
on:mousedown={(e) => onBoxMouseDown(e, i)}
>
<span class="tip"
style:display={dBox.warnLargeSize ? "block" : "none"}>
Warning: Region very large!
</span>
</div>
{/each}
</div>
</div>
</Block>
<style lang="scss">
.regions-container {
position: relative;
padding: 0;
}
.regions {
position: absolute;
left: 0px;
top: 0px;
width: 100%;
height: 100%;
.region {
position: absolute;
border: 2px solid blue;
background: teal;
z-index: var(--layer-3);
cursor: move;
.tip {
position: absolute;
left: 50%;
top: 50%;
transform: translate(-50%, -50%);
font-size: 12px;
font-weight: bold;
text-align: center;
color: red;
z-index: var(--layer-2);
}
}
}
</style>

36
src/lib/widgets/utils.ts
View File

@@ -36,3 +36,39 @@ export function isHidden(widget: IDragItem) {
return false;
}
export function generateBlankCanvas(width: number, height: number, fill: string = "#fff"): HTMLCanvasElement {
const canvas = document.createElement('canvas');
canvas.width = width;
canvas.height = height;
const ctx = canvas.getContext('2d');
ctx.save();
ctx.fillStyle = fill,
ctx.fillRect(0, 0, canvas.width, canvas.height);
ctx.restore();
return canvas;
}
export async function loadImage(imageURL: string): Promise<HTMLImageElement> {
return new Promise((resolve) => {
const e = new Image();
e.setAttribute('crossorigin', 'anonymous'); // Don't taint the canvas from loading files on-disk
e.addEventListener("load", () => { resolve(e); });
e.src = imageURL;
return e;
});
}
export async function generateImageCanvas(imageURL: string): Promise<[HTMLCanvasElement, number, number]> {
const image = await loadImage(imageURL);
const canvas = document.createElement('canvas');
canvas.width = image.width;
canvas.height = image.height;
const ctx = canvas.getContext('2d');
ctx.save();
ctx.fillStyle = "rgba(255, 255, 255, 0.0)";
ctx.fillRect(0, 0, canvas.width, canvas.height);
ctx.drawImage(image, 0, 0);
ctx.restore();
return [canvas, image.width, image.height];
}