Merge pull request #62 from space-nuko/area-widget

Regions widget

README.md

@@ -26,7 +26,7 @@ Also note that the saved workflow format is subject to change until it's been fi
 
 ## Features
 - **No-Code UI Builder** - A novel system for creating your own Stable Diffusion user interfaces from the basic components.
-- **Manage Multiple Workflows** - You can open as many workflows as you like and switch between them using tabs within the app.
+- **Manage Multiple Workflows** - You can open as many workflows as you like and switch between them using tabs within the app. You can also write custom "Send To" actions to copy your image outputs into other workflows for further processing.
 - **Use Your Existing Workflows** - Import workflows you've created in ComfyUI into ComfyBox and a new UI will be created for you.
 - **Extension Support** - All custom ComfyUI nodes are supported out of the box.
 - **Prompt Queue** - Queue up multiple prompts without waiting for them to finish first. Inspect currently queued and executed prompts.

src/lib/ComfyGraph.ts

@@ -11,7 +11,7 @@ import type { ComfyComboNode, ComfyWidgetNode } from "./nodes/widgets";
 import selectionState from "./stores/selectionState";
 import type { WritableLayoutStateStore } from "./stores/layoutStates";
 import layoutStates from "./stores/layoutStates";
-import type { ComfyWorkflow } from "./stores/workflowState";
+import type { ComfyWorkflow, WorkflowInstID } from "./stores/workflowState";
 import workflowState from "./stores/workflowState";
 
 type ComfyGraphEvents = {
@@ -163,7 +163,7 @@ export default class ComfyGraph extends LGraph {
                 // case node was cloned
                 const reg = LiteGraph.registered_node_types[node.type]
 
-                layoutState.groupItems(dragItemIDs, { title: reg.title })
+                layoutState.groupItems(dragItemIDs, { title: reg.title, variant: "accordion", openOnStartup: true })
             }
         }
 

src/lib/components/ComfyApp.ts

@@ -220,6 +220,7 @@ export default class ComfyApp {
             }
             await this.initDefaultWorkflow("defaultWorkflow", options);
             await this.initDefaultWorkflow("upscale", options);
+            await this.initDefaultWorkflow("conditioningRegions", options);
         }
 
         // Save current workflow automatically
@@ -356,6 +357,24 @@ export default class ComfyApp {
 
             ComfyApp.registerDefaultSlotHandlers(nodeId, nodeDef)
         }
+
+        ComfyApp.registerComfyBoxSlotTypes()
+    }
+
+    static registerComfyBoxSlotTypes() {
+        const reg = (type: string) => {
+            const lowerType = type.toLowerCase();
+            if (!LiteGraph.slot_types_in.includes(lowerType)) {
+                LiteGraph.slot_types_in.push(lowerType);
+            }
+            if (!LiteGraph.slot_types_out.includes(type)) {
+                LiteGraph.slot_types_out.push(type);
+            }
+        }
+
+        reg("COMFYBOX_IMAGE")
+        reg("COMFYBOX_IMAGES")
+        reg("COMFYBOX_REGION")
     }
 
     static registerDefaultSlotHandlers(nodeId: string, nodeDef: ComfyNodeDef) {
@@ -592,6 +611,7 @@ export default class ComfyApp {
 
         setColor("COMFYBOX_IMAGES", "rebeccapurple")
         setColor("COMFYBOX_IMAGE", "fuchsia")
+        setColor("COMFYBOX_REGION", "salmon")
         setColor(BuiltInSlotType.EVENT, "lightseagreen")
         setColor(BuiltInSlotType.ACTION, "lightseagreen")
     }
@@ -770,7 +790,9 @@ export default class ComfyApp {
 
         const promptFilename = get(configState).promptForWorkflowName;
 
-        let filename = "workflow.json";
+        const title = workflow.attrs.title.trim() || "workflow"
+
+        let filename = `${title}.json`;
         if (promptFilename) {
             filename = prompt("Save workflow as:", filename);
             if (!filename) return;
@@ -781,7 +803,7 @@ export default class ComfyApp {
         else {
             const date = new Date();
             const formattedDate = date.toISOString().replace(/:/g, '-').replace(/\.\d{3}/g, '').replace('T', '_').replace("Z", "");
-            filename = `workflow - ${formattedDate}.json`
+            filename = `${title} - ${formattedDate}.json`
         }
 
         const indent = 2

src/lib/components/ComfyQueue.svelte

@@ -150,7 +150,7 @@
      let submessage = `Nodes: ${Object.keys(entry.prompt).length}`
 
      if (Object.keys(entry.outputs).length > 0) {
-         const imageCount = Object.values(entry.outputs).flatMap(o => o.images).length
+         const imageCount = Object.values(entry.outputs).filter(o => o.images).flatMap(o => o.images).length
          submessage = `Images: ${imageCount}`
      }
 
@@ -172,13 +172,23 @@
          result.images = thumbnails.map(convertComfyOutputToComfyURL);
      }
 
+     const outputs = Object.values(entry.outputs)
+                           .filter(o => o.images)
+                           .flatMap(o => o.images)
+                           .map(convertComfyOutputToComfyURL);
+     if (outputs) {
+         result.images = result.images.concat(outputs)
+     }
+
      return result;
  }
 
  function convertCompletedEntry(entry: CompletedQueueEntry): QueueUIEntry {
      const result = convertEntry(entry.entry, entry.status);
 
-     const images = Object.values(entry.entry.outputs).flatMap(o => o.images)
+     const images = Object.values(entry.entry.outputs)
+                          .filter(o => o.images)
+                          .flatMap(o => o.images)
                           .map(convertComfyOutputToComfyURL);
      result.images = images
 

src/lib/components/WidgetContainer.svelte

@@ -151,5 +151,7 @@
 
  .edit {
      border: 2px dashed var(--color-blue-400);
+     pointer-events: none;
+     user-select: none;
  }
 </style>

src/lib/components/gradio/gallery/Gallery.svelte

@@ -15,6 +15,7 @@
 	export let label: string;
 	export let root: string = "";
 	export let root_url: null | string = null;
+    export let scrollOnUpdate = false;
 	export let value: Array<string> | Array<FileData> | null = null;
 	export let style: Styles = {
 		grid_cols: [2],
@@ -120,6 +121,7 @@
 	let container: HTMLDivElement;
 
 	async function scroll_to_img(index: number | null) {
+        if (!scrollOnUpdate) return;
 		if (typeof index !== "number") return;
 		await tick();
 

src/lib/nodes/ComfyRegionToCoordsNode.ts

@@ -0,0 +1,35 @@
+import { LiteGraph, type SlotLayout } from "@litegraph-ts/core";
+import ComfyGraphNode from "./ComfyGraphNode";
+
+export default class ComfyRegionToCoordsNode extends ComfyGraphNode {
+    static slotLayout: SlotLayout = {
+        inputs: [
+            { name: "in", type: "COMFYBOX_REGION" },
+        ],
+        outputs: [
+            // same order as conditioning nodes
+            { name: "width", type: "number" },
+            { name: "height", type: "number" },
+            { name: "x", type: "number" },
+            { name: "y", type: "number" },
+        ],
+    }
+
+    override onExecute() {
+        const value = this.getInputData(0);
+        if (!Array.isArray(value))
+            return;
+
+        this.setOutputData(0, value[2])
+        this.setOutputData(1, value[3])
+        this.setOutputData(2, value[0])
+        this.setOutputData(3, value[1])
+    }
+}
+
+LiteGraph.registerNodeType({
+    class: ComfyRegionToCoordsNode,
+    title: "Comfy.RegionToCoords",
+    desc: "Converts a COMFYBOX_REGION to four outputs of [width, height, x, y]",
+    type: "utils/region_to_coords"
+})

src/lib/nodes/index.ts

@@ -1,5 +1,3 @@
-import "$lib/nodes/ComfyGraphNode";
-
 export { default as ComfyReroute } from "./ComfyReroute"
 export { default as ComfyPickFirstNode } from "./ComfyPickFirstNode"
 export { default as ComfyValueControl } from "./ComfyValueControl"
@@ -8,3 +6,4 @@ export { default as ComfyTriggerNewEventNode } from "./ComfyTriggerNewEventNode"
 export { default as ComfyConfigureQueuePromptButton } from "./ComfyConfigureQueuePromptButton"
 export { default as ComfyPickImageNode } from "./ComfyPickImageNode"
 export { default as ComfyNoChangeEvent } from "./ComfyNoChangeEvent"
+export { default as ComfyRegionToCoordsNode } from "./ComfyRegionToCoordsNode"

src/lib/nodes/widgets/ComfyMultiRegionNode.ts

@@ -0,0 +1,236 @@
+import { BuiltInSlotType, LiteGraph, type IComboWidget, type SlotLayout, type INumberWidget } from "@litegraph-ts/core";
+
+import MultiRegionWidget from "$lib/widgets/MultiRegionWidget.svelte";
+import ComfyWidgetNode, { type ComfyWidgetProperties } from "./ComfyWidgetNode";
+import { clamp, isComfyBoxImageMetadata, type ComfyBoxImageMetadata, comfyBoxImageToComfyURL } from "$lib/utils";
+import { writable, type Writable } from "svelte/store";
+
+/* x, y, width, height, all in range 0.0 - 1.0 */
+export type BoundingBox = [number, number, number, number]
+
+function isBoundingBox(param: any): param is BoundingBox {
+    return Array.isArray(param) && param.length === 4 && param.every(i => typeof i === "number")
+}
+
+export interface ComfyMultiRegionProperties extends ComfyWidgetProperties {
+    regionCount: number,
+    canvasWidth: number,
+    canvasHeight: number,
+    canvasImageURL: string | null,
+    inputType: "size" | "image"
+}
+
+const DEFAULT_BBOX: BoundingBox = [0.4, 0.4, 0.2, 0.2];
+
+export default class ComfyMultiRegionNode extends ComfyWidgetNode<BoundingBox[]> {
+    override properties: ComfyMultiRegionProperties = {
+        tags: [],
+        defaultValue: false,
+        regionCount: 1,
+        canvasWidth: 512,
+        canvasHeight: 512,
+        canvasImageURL: null,
+        inputType: "size"
+    }
+
+    static slotLayout: SlotLayout = {
+        inputs: [
+            { name: "store", type: BuiltInSlotType.ACTION },
+        ],
+        outputs: [
+            { name: "changed", type: BuiltInSlotType.EVENT },
+
+            // dynamic outputs, may be removed later
+            { name: "region_1", type: "COMFYBOX_REGION" },
+        ]
+    }
+
+    override svelteComponentType = MultiRegionWidget;
+    override defaultValue: BoundingBox[] = [[...DEFAULT_BBOX]];
+    override outputSlotName = null;
+    override storeActionName = "store";
+    override changedEventName = "changed";
+
+    sizeChanged: Writable<boolean> = writable(true);
+    regionsChanged: Writable<boolean> = writable(true);
+
+    inputTypeWidget: IComboWidget
+    regionCountWidget: INumberWidget
+
+    constructor(name?: string) {
+        super(name, [[...DEFAULT_BBOX]])
+
+        this.inputTypeWidget = this.addWidget("combo", "Input Type", this.properties.inputType, "inputType", { values: ["size", "image"] })
+        this.regionCountWidget = this.addWidget(
+            "number",
+            "# of Regions",
+            this.properties.regionCount,
+            (v: number) => {
+                this.setProperty("regionCount", clamp(v, 1, 16));
+            },
+            { min: 1, max: 16, step: 1, precision: 0 }
+        )
+    }
+
+    override onPropertyChanged(property: any, value: any) {
+        if (property === "regionCount") {
+            this.updateRegions()
+        }
+        else if (property === "width" || property === "height") {
+            this.updateSize();
+        }
+        else if (property === "inputType") {
+            this.updateInputType();
+        }
+    }
+
+    override onExecute() {
+        let width = 0;
+        let height = 0;
+        let imageURL: string | null = null;
+
+        this.updateInputType();
+
+        if (this.properties.inputType === "image") {
+            let comfyBoxImage = this.getInputData(1);
+            if (isComfyBoxImageMetadata(comfyBoxImage)) {
+                imageURL = comfyBoxImageToComfyURL(comfyBoxImage)
+                width = comfyBoxImage.width;
+                height = comfyBoxImage.height;
+            }
+            else {
+                imageURL = null;
+            }
+        }
+        else if (this.properties.inputType === "size") {
+            width = this.getInputData(1) || 0
+            height = this.getInputData(2) || 0
+        }
+
+        if (width != this.properties.canvasWidth || height != this.properties.canvasHeight || imageURL != this.properties.canvasImageURL) {
+            this.properties.canvasWidth = width;
+            this.properties.canvasHeight = height;
+            this.properties.canvasImageURL = imageURL
+            this.updateSize();
+        }
+
+        const value = this.getValue();
+
+        for (let index = 0; index < this.properties.regionCount; index++) {
+            const bbox = value[index]
+            if (bbox != null) {
+                const output = this.outputs[index + 1] // + changed slot
+                if (output != null) {
+                    let data = this.getOutputData(index + 1) || [0, 0, 0, 0]
+                    data[0] = bbox[0] * this.properties.canvasWidth
+                    data[1] = bbox[1] * this.properties.canvasHeight
+                    data[2] = bbox[2] * this.properties.canvasWidth
+                    data[3] = bbox[3] * this.properties.canvasHeight
+                    this.setOutputData(index + 1, data)
+                }
+            }
+        }
+    }
+
+    private updateRegions() {
+        this.properties.regionCount = Math.max(this.properties.regionCount, 0);
+
+        if (this.outputs.length === this.properties.regionCount + 1)
+            return
+
+        for (let index = this.outputs.length - 1; index > this.properties.regionCount; index--) {
+            if (this.outputs[index].type !== BuiltInSlotType.EVENT) {
+                this.removeOutput(index);
+            }
+        }
+
+        for (let index = this.outputs.length - 1; index < this.properties.regionCount; index++) {
+            this.addOutput(`region_${index + 1}`, "COMFYBOX_REGION")
+        }
+
+        this.regionsChanged.set(true);
+        // this.notifyPropsChanged();
+
+        this.setValue(this.getValue())
+    }
+
+    private _prevWidth = null
+    private _prevHeight = null
+    private _prevImageURL = null
+
+    private updateSize(value?: BoundingBox[]): BoundingBox[] {
+        this.properties.canvasWidth = Math.max(this.properties.canvasWidth, 0);
+        this.properties.canvasHeight = Math.max(this.properties.canvasHeight, 0);
+
+        value ||= this.getValue();
+
+        for (const bbox of value) {
+            bbox[0] = clamp(bbox[0], 0, 1);
+            bbox[1] = clamp(bbox[1], 0, 1);
+            bbox[2] = clamp(bbox[2], 0, 1)
+            bbox[3] = clamp(bbox[3], 0, 1)
+        }
+
+        const sizeChanged = this.properties.canvasWidth != this._prevWidth
+            || this.properties.canvasHeight != this._prevHeight
+            || this.properties.canvasImageURL != this._prevImageURL
+
+        this.sizeChanged.set(sizeChanged);
+
+        this._prevWidth = this.properties.canvasWidth;
+        this._prevHeight = this.properties.canvasHeight;
+        this._prevImageURL = this.properties.canvasImageURL;
+
+        // this.notifyPropsChanged();
+
+        return value
+    }
+
+    private updateInputType() {
+        const inputType = this.properties.inputType;
+        if (inputType === "image" && this.inputs[1]?.name === "image")
+            return
+        else if (inputType === "size" && this.inputs[1]?.name === "width")
+            return
+
+        for (let index = this.inputs.length - 1; index >= 1; index--) {
+            if (this.inputs[index].type !== BuiltInSlotType.ACTION) {
+                this.removeInput(index);
+            }
+        }
+
+        if (inputType === "image") {
+            this.addInput("image", "COMFYBOX_IMAGE");
+        }
+        else if (inputType === "size") {
+            this.addInput("width", "number")
+            this.addInput("height", "number")
+        }
+    }
+
+    override parseValue(param: any): BoundingBox[] {
+        if (param == null || this.properties.regionCount <= 0)
+            return []
+
+        let val = []
+
+        if (isBoundingBox(param))
+            val = this.updateSize([param])
+
+        if (Array.isArray(param) && param.every(isBoundingBox))
+            val = this.updateSize(param.splice(0, this.properties.regionCount))
+
+        // Fill the array with missing regions
+        for (let index = val.length; index < this.properties.regionCount; index++)
+            val.push([...DEFAULT_BBOX])
+
+        return val;
+    }
+}
+
+LiteGraph.registerNodeType({
+    class: ComfyMultiRegionNode,
+    title: "UI.MultiRegion",
+    desc: "Overlays one or more regions over a canvas of the given width/height",
+    type: "ui/multi_region"
+})

src/lib/nodes/widgets/index.ts

@@ -8,3 +8,4 @@ export { default as ComfyImageUploadNode } from "./ComfyImageUploadNode"
 export { default as ComfyRadioNode } from "./ComfyRadioNode"
 export { default as ComfyNumberNode } from "./ComfyNumberNode"
 export { default as ComfyTextNode } from "./ComfyTextNode"
+export { default as ComfyMultiRegionNode } from "./ComfyMultiRegionNode"

src/lib/widgets/ImageUploadWidget.svelte

@@ -15,6 +15,7 @@
  import NumberInput from "$lib/components/NumberInput.svelte";
 	import type { ComfyImageEditorNode } from "$lib/nodes/widgets";
 	import { ImageViewer } from "$lib/ImageViewer";
+	import { generateBlankCanvas, generateImageCanvas } from "./utils";
 
  export let widget: WidgetLayout | null = null;
  export let isMobile: boolean = false;
@@ -55,42 +56,6 @@
      showModal = false;
  }
 
- function generateBlankCanvas(width: number, height: number, fill: string = "#fff"): HTMLCanvasElement {
-     const canvas = document.createElement('canvas');
-     canvas.width = width;
-     canvas.height = height;
-     const ctx = canvas.getContext('2d');
-     ctx.save();
-     ctx.fillStyle = fill,
-           ctx.fillRect(0, 0, canvas.width, canvas.height);
-     ctx.restore();
-     return canvas;
- }
-
- async function loadImage(imageURL: string): Promise<HTMLImageElement> {
-     return new Promise((resolve) => {
-         const e = new Image();
-         e.setAttribute('crossorigin', 'anonymous'); // Don't taint the canvas from loading files on-disk
-         e.addEventListener("load", () => { resolve(e); });
-         e.src = imageURL;
-         return e;
-     });
- }
-
- async function generateImageCanvas(imageURL: string): Promise<[HTMLCanvasElement, number, number]> {
-     const image = await loadImage(imageURL);
-     const canvas = document.createElement('canvas');
-     canvas.width = image.width;
-     canvas.height = image.height;
-     const ctx = canvas.getContext('2d');
-     ctx.save();
-     ctx.fillStyle = "rgba(255, 255, 255, 0.0)";
-     ctx.fillRect(0, 0, canvas.width, canvas.height);
-     ctx.drawImage(image, 0, 0);
-     ctx.restore();
-     return [canvas, image.width, image.height];
- }
-
  const FILENAME: string = "ComfyUITemp.png";
  const SUBFOLDER: string = "ComfyBox_Editor";
  const DIRECTORY: ComfyUploadImageType = "input";

src/lib/widgets/MultiRegionWidget.svelte

@@ -0,0 +1,616 @@
+<script lang="ts">
+ /*
+  * Portions of this code were adapted from multidiffusion-upscaler-for-automatic1111 ver. 2023.3.28
+  * which was licensed under GPL-3. Subsequent versions were relicensed as CC BY-NC-SA 4.0
+  * https://github.com/pkuliyi2015/multidiffusion-upscaler-for-automatic1111/commit/e116fe946fe44c52d5500ace9ce65d4d685cff46
+  */
+ import { onMount } from "svelte";
+ import type { ComfyMultiRegionNode } from "$lib/nodes/widgets";
+ import type { BoundingBox } from "$lib/nodes/widgets/ComfyMultiRegionNode";
+ import type { WidgetLayout } from "$lib/stores/layoutStates";
+ import { Block, BlockLabel } from "@gradio/atoms";
+ import { Chart as ChartIcon } from "@gradio/icons";
+ import { Range } from "@gradio/form";
+ import { writable, type Writable } from "svelte/store";
+ import { generateBlankCanvas, generateImageCanvas, loadImage } from "./utils";
+ import { clamp } from "$lib/utils";
+ import Row from "$lib/components/gradio/app/Row.svelte";
+
+// ref: https://html-color.codes/
+const COLOR_MAP: [string, string][] = [
+    ['#ff0000', 'rgba(255, 0, 0, 0.3)'],          // red
+    ['#ff9900', 'rgba(255, 153, 0, 0.3)'],        // orange
+    ['#ffff00', 'rgba(255, 255, 0, 0.3)'],        // yellow
+    ['#33cc33', 'rgba(51, 204, 51, 0.3)'],        // green
+    ['#33cccc', 'rgba(51, 204, 204, 0.3)'],       // indigo
+    ['#0066ff', 'rgba(0, 102, 255, 0.3)'],        // blue
+    ['#6600ff', 'rgba(102, 0, 255, 0.3)'],        // purple
+    ['#cc00cc', 'rgba(204, 0, 204, 0.3)'],        // dark pink
+    ['#ff6666', 'rgba(255, 102, 102, 0.3)'],      // light red
+    ['#ffcc66', 'rgba(255, 204, 102, 0.3)'],      // light orange
+    ['#99cc00', 'rgba(153, 204, 0, 0.3)'],        // lime green
+    ['#00cc99', 'rgba(0, 204, 153, 0.3)'],        // teal
+    ['#0099cc', 'rgba(0, 153, 204, 0.3)'],        // steel blue
+    ['#9933cc', 'rgba(153, 51, 204, 0.3)'],       // lavender
+    ['#ff3399', 'rgba(255, 51, 153, 0.3)'],       // hot pink
+    ['#996633', 'rgba(153, 102, 51, 0.3)'],       // brown
+];
+
+ export let widget: WidgetLayout | null = null;
+ export let isMobile: boolean = false;
+
+ let imageContainer: HTMLDivElement | null;
+ let imageElem: HTMLImageElement | null;
+
+ let node: ComfyMultiRegionNode | null = null;
+ let nodeValue: Writable<BoundingBox[]> = writable([]);
+ let sizeChanged: Writable<boolean> = writable(false);
+ let regionsChanged: Writable<boolean> = writable(false);
+ let propsChanged: Writable<number> = writable(0);
+ let selectedIndex: number = 0;
+ let imageOpacity: number = 1;
+
+ $: widget && setNodeValue(widget);
+
+ function setNodeValue(widget: WidgetLayout) {
+     if (widget) {
+         node = widget.node as ComfyMultiRegionNode
+         nodeValue = node.value;
+         propsChanged = node.propsChanged;
+         sizeChanged = node.sizeChanged;
+         regionsChanged = node.regionsChanged;
+
+         updateImageAndDBoxes();
+     }
+ };
+
+ let showWidget: boolean = false;
+
+ if (sizeChanged && $sizeChanged) {
+     console.error("Z@");
+ }
+
+ type DisplayBoundingBox = {
+     xPx: number,
+     yPx: number,
+     widthPx: number,
+     heightPx: number,
+     warnLargeSize: boolean,
+     bgColor: string,
+     borderColor: string
+ }
+
+ let displayBoxes = [];
+
+ async function recreateDisplayBoxes(_node?: ComfyMultiRegionNode, bboxes?: BoundingBox[]): Promise<DisplayBoundingBox[]> {
+     _node ||= node;
+     bboxes ||= $nodeValue
+
+     console.debug("[MultiRegionWidget] Recreate!", bboxes, imageElem, _node)
+
+     if (_node != null && imageElem != null && imageContainer != null) {
+         selectedIndex = clamp(selectedIndex, 0, bboxes.length - 1);
+         await updateImage(_node.properties.canvasWidth, _node.properties.canvasHeight, _node.properties.canvasImageURL);
+         return bboxes.map((b, i) => displayBoundingBox(b, i, imageElem))
+     }
+     else {
+         return []
+     }
+ }
+
+ async function updateImageAndDBoxes() {
+     if (node == null)
+         return;
+
+     return updateImage(node.properties.canvasWidth, node.properties.canvasHeight, node.properties.canvasImageURL)
+         .then(() => {
+             return recreateDisplayBoxes()
+         })
+         .then(dbs => {
+             displayBoxes = dbs;
+         })
+ }
+
+ $: if (node != null && $sizeChanged) {
+     updateImageAndDBoxes();
+ }
+
+ onMount(async () => {
+     await updateImageAndDBoxes();
+ })
+
+ $: if ($regionsChanged) {
+     $regionsChanged = false;
+     recreateDisplayBoxes(node, $nodeValue).then(dbs => displayBoxes = dbs);
+ }
+
+ let hasImage = false;
+
+ async function updateImage(width: number, height: number, imageURL: string | null) {
+     showWidget = width > 0 && height > 0;
+     hasImage = imageURL != null
+
+     if (imageURL == null) {
+         const blank = generateBlankCanvas(width, height, "transparent");
+         imageURL = blank.toDataURL();
+     }
+
+     const newImg = await loadImage(imageURL);
+     newImg.classList.add("regions-image");
+     if (imageContainer != null) {
+         imageContainer.replaceChildren(newImg)
+     }
+     imageElem = newImg;
+     imageElem.style.border = `${BORDER_SIZE_PX}px solid var(--border-color-primary)`;
+
+     $sizeChanged = false;
+ }
+
+ const BBOX_WARNING_SIZE = 1280;
+
+ function onBBoxMoved(index: number, newBBox: BoundingBox) {
+     $nodeValue[index] = newBBox;
+     displayBoxes[index] = displayBoundingBox(newBBox, index, imageElem, displayBoxes[index]);
+ }
+
+ function displayBoundingBox(bbox: BoundingBox, index: number, imageElem: HTMLImageElement, out?: DisplayBoundingBox): DisplayBoundingBox {
+     const [x, y, w, h] = bbox;
+
+     // client: image widget display size
+     // natural: content image real size
+     const vpScale = Math.min(imageElem.clientWidth / imageElem.naturalWidth, imageElem.clientHeight / imageElem.naturalHeight);
+     const imageElemCenterX = (imageElem.clientWidth)  / 2;
+     const imageElemCenterY = (imageElem.clientHeight) / 2;
+     const scaledX = imageElem.naturalWidth  * vpScale;
+     const scaledY = imageElem.naturalHeight * vpScale;
+     const viewRectLeft  = imageElemCenterX - scaledX / 2 + BORDER_SIZE_PX;
+     const viewRectRight = imageElemCenterX + scaledX / 2 + BORDER_SIZE_PX;
+     const viewRectTop   = imageElemCenterY - scaledY / 2 + BORDER_SIZE_PX;
+     const viewRectDown  = imageElemCenterY + scaledY / 2 + BORDER_SIZE_PX;
+
+     const xDiv = viewRectLeft + scaledX * x;
+     const yDiv = viewRectTop  + scaledY * y;
+     const wDiv = Math.min(scaledX * w, viewRectRight - xDiv);
+     const hDiv = Math.min(scaledY * h, viewRectDown - yDiv);
+
+     // Calculate warning bbox size
+     const upscalerFactor = 1.0;
+     // if (!is_t2i) {
+     //     upscalerFactor = getUpscalerFactor();
+     // }
+     const maxSize = BBOX_WARNING_SIZE / upscalerFactor * vpScale;
+     const maxW = maxSize / scaledX;
+     const maxH = maxSize / scaledY;
+     const warnLargeSize = w > maxW || h > maxH
+
+     const [borderColor, bgColor] = COLOR_MAP[index % COLOR_MAP.length]
+
+     out ||= {} as DisplayBoundingBox
+     out.xPx = xDiv;
+     out.yPx = yDiv;
+     out.widthPx = wDiv;
+     out.heightPx = hDiv;
+     out.warnLargeSize = warnLargeSize;
+     out.bgColor = bgColor
+     out.borderColor = borderColor
+
+     return out;
+ }
+
+ const RESIZE_BORDER = 5;
+ const MOVE_BORDER = 5;
+ const BORDER_SIZE_PX = 3;
+
+ function updateCursorStyle(e: MouseEvent) {
+     // This function changes the cursor style when hovering over the bounding box
+     const div = e.target as HTMLDivElement;
+     const boxRect = div.getBoundingClientRect();
+     const mouseX = e.clientX;
+     const mouseY = e.clientY;
+
+     const resizeLeft   = mouseX >= boxRect.left && mouseX <= boxRect.left + RESIZE_BORDER;
+     const resizeRight  = mouseX >= boxRect.right - RESIZE_BORDER && mouseX <= boxRect.right;
+     const resizeTop    = mouseY >= boxRect.top && mouseY <= boxRect.top + RESIZE_BORDER;
+     const resizeBottom = mouseY >= boxRect.bottom - RESIZE_BORDER && mouseY <= boxRect.bottom;
+
+     if ((resizeLeft && resizeTop) || (resizeRight && resizeBottom)) {
+         div.style.cursor = 'nwse-resize';
+     } else if ((resizeLeft && resizeBottom) || (resizeRight && resizeTop)) {
+         div.style.cursor = 'nesw-resize';
+     } else if (resizeLeft || resizeRight) {
+         div.style.cursor = 'ew-resize';
+     } else if (resizeTop || resizeBottom) {
+         div.style.cursor = 'ns-resize';
+     } else {
+         div.style.cursor = 'move';
+     }
+ }
+
+ function onBoxMouseDown(e: MouseEvent, index: number) {
+     if (e.button !== 0)
+         return;
+
+     // Get the bounding box
+     let bbox = $nodeValue[index];
+     if (!imageElem || !bbox)
+         return;
+
+     selectedIndex = index;
+
+     // Check if the click is inside the bounding box
+     const div = e.target as HTMLDivElement;
+     const boxRect = div.getBoundingClientRect();
+     let mouseX = e.clientX;
+     let mouseY = e.clientY;
+
+     const resizeLeft   = mouseX >= boxRect.left && mouseX <= boxRect.left + RESIZE_BORDER;
+     const resizeRight  = mouseX >= boxRect.right - RESIZE_BORDER && mouseX <= boxRect.right;
+     const resizeTop    = mouseY >= boxRect.top && mouseY <= boxRect.top + RESIZE_BORDER;
+     const resizeBottom = mouseY >= boxRect.bottom - RESIZE_BORDER && mouseY <= boxRect.bottom;
+
+     const moveHorizontal = mouseX >= boxRect.left + MOVE_BORDER && mouseX <= boxRect.right  - MOVE_BORDER;
+     const moveVertical   = mouseY >= boxRect.top  + MOVE_BORDER && mouseY <= boxRect.bottom - MOVE_BORDER;
+
+     if (!resizeLeft && !resizeRight && !resizeTop && !resizeBottom && !moveHorizontal && !moveVertical)
+         return;
+
+     const horizontalPivot = resizeLeft ? bbox[0] + bbox[2] : bbox[0];
+     const verticalPivot   = resizeTop  ? bbox[1] + bbox[3] : bbox[1];
+
+     // Canvas can be regarded as invariant during the drag operation
+     // Calculate in advance to reduce overhead
+
+     // Calculate viewport scale based on the current canvas size and the natural image size
+     let vpScale = Math.min(imageElem.clientWidth / imageElem.naturalWidth, imageElem.clientHeight / imageElem.naturalHeight);
+     let vpOffset = imageElem.getBoundingClientRect();
+
+     // Calculate scaled dimensions of the canvas
+     let scaledX = imageElem.naturalWidth * vpScale;
+     let scaledY = imageElem.naturalHeight * vpScale;
+
+     // Calculate the canvas center and view rectangle coordinates
+     let canvasCenterX = (vpOffset.left + window.scrollX) + imageElem.clientWidth  / 2;
+     let canvasCenterY = (vpOffset.top  + window.scrollY) + imageElem.clientHeight / 2;
+     let viewRectLeft  = canvasCenterX - scaledX / 2 - window.scrollX;
+     let viewRectRight = canvasCenterX + scaledX / 2 - window.scrollX;
+     let viewRectTop   = canvasCenterY - scaledY / 2 - window.scrollY;
+     let viewRectDown  = canvasCenterY + scaledY / 2 - window.scrollY;
+
+     mouseX = Math.min(Math.max(mouseX, viewRectLeft), viewRectRight);
+     mouseY = Math.min(Math.max(mouseY, viewRectTop),  viewRectDown);
+
+     // Move or resize the bounding box on mousemove
+     function onMouseMove(e) {
+         // Prevent selecting anything irrelevant
+         e.preventDefault();
+
+         // Get the new mouse position
+         let newMouseX = e.clientX;
+         let newMouseY = e.clientY;
+
+         // clamp the mouse position to the view rectangle
+         newMouseX = Math.min(Math.max(newMouseX, viewRectLeft), viewRectRight);
+         newMouseY = Math.min(Math.max(newMouseY, viewRectTop),  viewRectDown);
+
+         // Calculate the mouse movement delta
+         const dx = (newMouseX - mouseX) / scaledX;
+         const dy = (newMouseY - mouseY) / scaledY;
+
+         // Update the mouse position
+         mouseX = newMouseX;
+         mouseY = newMouseY;
+
+         // if no move just return
+         if (dx === 0 && dy === 0) { return; }
+
+         // Update the mouse position
+         let [x, y, w, h] = bbox;
+
+         if (moveHorizontal && moveVertical) {
+             // If moving the bounding box
+             x = Math.min(Math.max(x + dx, 0), 1 - w);
+             y = Math.min(Math.max(y + dy, 0), 1 - h);
+         } else {
+             // If resizing the bounding box
+             if (resizeLeft || resizeRight) {
+                 if (x < horizontalPivot) {
+                     if (dx <= w) {
+                         // If still within the left side of the pivot
+                         x = x + dx;
+                         w = w - dx;
+                     } else {
+                         // If crossing the pivot
+                         w = dx - w;
+                         x = horizontalPivot;
+                     }
+                 } else {
+                     if (w + dx < 0) {
+                         // If still within the right side of the pivot
+                         x = horizontalPivot + w + dx;
+                         w = - dx - w;
+                     } else {
+                         // If crossing the pivot
+                         x = horizontalPivot;
+                         w = w + dx;
+                     }
+                 }
+
+                 // Clamp the bounding box to the image
+                 if (x < 0) {
+                     w = w + x;
+                     x = 0;
+                 } else if (x + w > 1) {
+                     w = 1 - x;
+                 }
+             }
+             // Same as above, but for the vertical axis
+             if (resizeTop || resizeBottom) {
+                 if (y < verticalPivot) {
+                     if (dy <= h) {
+                         y = y + dy;
+                         h = h - dy;
+                     } else {
+                         h = dy - h;
+                         y = verticalPivot;
+                     }
+                 } else {
+                     if (h + dy < 0) {
+                         y = verticalPivot + h + dy;
+                         h = - dy - h;
+                     } else {
+                         y = verticalPivot;
+                         h = h + dy;
+                     }
+                 }
+                 if (y < 0) {
+                     h = h + y;
+                     y = 0;
+                 } else if (y + h > 1) {
+                     h = 1 - y;
+                 }
+             }
+         }
+         const [div, old_bbox, _] = $nodeValue[index];
+
+         // If all the values are the same, just return
+         if (old_bbox[0] === x && old_bbox[1] === y && old_bbox[2] === w && old_bbox[3] === h) { return; }
+
+         // else update the bbox
+         bbox[0] = x;
+         bbox[1] = y;
+         bbox[2] = w;
+         bbox[3] = h;
+         onBBoxMoved(index, bbox);
+     }
+
+     const onMouseUp = () => {
+         document.removeEventListener('mousemove', onMouseMove);
+         document.removeEventListener('mouseup', onMouseUp);
+         $regionsChanged = true;
+         $nodeValue = $nodeValue;
+     }
+
+     // Add the event listeners
+     document.addEventListener('mousemove', onMouseMove);
+     document.addEventListener('mouseup', onMouseUp);
+ }
+
+ async function onResize() {
+     displayBoxes = await recreateDisplayBoxes();
+ }
+
+ function updateSelectedIndex(newIndexPlusOne: number) {
+     selectedIndex = clamp(newIndexPlusOne - 1, 0, $nodeValue.length - 1);
+ }
+
+ function updateX(newX: number) {
+     const bbox = $nodeValue[selectedIndex]
+     const dbox = displayBoxes[selectedIndex]
+     if (!bbox || !dbox)
+         return
+
+     bbox[0] = newX
+     displayBoxes[selectedIndex] = displayBoundingBox(bbox, selectedIndex, imageElem, dbox);
+ }
+
+ function updateY(newY: number) {
+     const bbox = $nodeValue[selectedIndex]
+     const dbox = displayBoxes[selectedIndex]
+     if (!bbox || !dbox)
+         return
+
+     bbox[1] = newY
+     displayBoxes[selectedIndex] = displayBoundingBox(bbox, selectedIndex, imageElem, dbox);
+ }
+
+ function updateWidth(newWidth: number) {
+     const bbox = $nodeValue[selectedIndex]
+     const dbox = displayBoxes[selectedIndex]
+     if (!bbox || !dbox)
+         return
+
+     bbox[2] = newWidth
+     displayBoxes[selectedIndex] = displayBoundingBox(bbox, selectedIndex, imageElem, dbox);
+ }
+
+ function updateHeight(newHeight: number) {
+     const bbox = $nodeValue[selectedIndex]
+     const dbox = displayBoxes[selectedIndex]
+     if (!bbox || !dbox)
+         return
+
+     bbox[3] = newHeight
+     displayBoxes[selectedIndex] = displayBoundingBox(bbox, selectedIndex, imageElem, dbox);
+ }
+
+ async function updateValue() {
+     // Clamp regions
+     const bbox = $nodeValue[selectedIndex]
+     const dbox = displayBoxes[selectedIndex]
+     if (bbox && dbox) {
+         bbox[2] = clamp(bbox[2], 0, 1)
+         bbox[3] = clamp(bbox[3], 0, 1)
+         bbox[0] = clamp(bbox[0], 0, 1 - bbox[2])
+         bbox[1] = clamp(bbox[1], 0, 1 - bbox[3])
+         displayBoxes[selectedIndex] = displayBoundingBox(bbox, selectedIndex, imageElem, dbox);
+     }
+
+     await updateImageAndDBoxes();
+
+     // Force reactivity after changing a bbox's internal values
+     $nodeValue = $nodeValue
+ }
+</script>
+
+<svelte:window on:resize={onResize}/>
+
+{#key node?.properties.canvasWidth}
+    <Block>
+        {#if widget?.attrs.title}
+            {@const label = widget.attrs.title}
+            <BlockLabel
+                label={label}
+                show_label={label != ""}
+                Icon={ChartIcon}
+                float={label != ""}
+            />
+        {/if}
+        {#if showWidget}
+            {@const selectedBBox = $nodeValue[selectedIndex]}
+            <div class="regions-container">
+                <div bind:this={imageContainer} class="regions-image-container"
+                     style:opacity="{hasImage ? imageOpacity * 100 : 100}%">
+                    <img bind:this={imageElem} class="regions-image" />
+                </div>
+                <div class="regions">
+                    {#each displayBoxes as dBox, i}
+                        {@const selected = selectedIndex === i}
+                        <div class="region"
+                             style:left="{dBox.xPx}px"
+                             style:top="{dBox.yPx}px"
+                             style:width="{dBox.widthPx}px"
+                             style:height="{dBox.heightPx}px"
+                             style:background={dBox.bgColor}
+                             style:border-style={selected ? "solid" : "dotted"}
+                             style:border-color={dBox.borderColor}
+                             style:display="block"
+                             style:opacity={selected ? "100%" : "40%"}
+                             style:z-index={selected ? "var(--layer-3)" : "var(--layer-2)"}
+                             on:mousemove={updateCursorStyle}
+                             on:mousedown={(e) => onBoxMouseDown(e, i)}
+                            >
+                            <span class="tip"
+                                  style:display={dBox.warnLargeSize ? "block" : "none"}>
+                                Warning: Region very large!
+                            </span>
+                        </div>
+                    {/each}
+                </div>
+            </div>
+            {#if selectedBBox}
+                <Block>
+                    <Row>
+                        <Range label="Region #" value={selectedIndex+1}
+                                show_label={true} minimum={1} maximum={$nodeValue.length} step={1}
+                                on:change={(e) => updateSelectedIndex(e.detail)}
+                            on:release={updateValue}
+                            />
+                    </Row>
+                    <Row>
+                        <Range label="X" value={selectedBBox[0]}
+                               show_label={true} minimum={0.0} maximum={1.0} step={0.01}
+                               on:change={(e) => updateX(e.detail)}
+                            on:release={updateValue}
+                            />
+                            <Range label="Width" value={selectedBBox[2]}
+                                   show_label={true} minimum={0.0} maximum={1.0} step={0.01}
+                                   on:change={(e) => updateWidth(e.detail)}
+                                on:release={updateValue}
+                                />
+                    </Row>
+                    <Row>
+                        <Range label="Y" value={selectedBBox[1]}
+                               show_label={true} minimum={0.0} maximum={1.0} step={0.01}
+                               on:change={(e) => updateY(e.detail)}
+                            on:release={updateValue}
+                            />
+                            <Range label="Height" value={selectedBBox[3]}
+                                   show_label={true} minimum={0.0} maximum={1.0} step={0.01}
+                                   on:change={(e) => updateHeight(e.detail)}
+                                on:release={updateValue}
+                                />
+                    </Row>
+                    {#if hasImage}
+                        <Row>
+                            <Range label="Image Opacity" bind:value={imageOpacity}
+                                   show_label={true} minimum={0.0} maximum={1.0} step={0.01}/>
+                        </Row>
+                    {/if}
+                </Block>
+            {/if}
+        {:else}
+            <div class="regions-empty">
+                <span>(No regions)</span>
+            </div>
+        {/if}
+    </Block>
+{/key}
+
+<style lang="scss">
+ .regions-container {
+     position: relative;
+     padding: 0;
+
+     .regions-image-container {
+         display: flex;
+
+         img {
+             height: 100%;
+             border: 3px solid var(--input-border-color);
+         }
+     }
+ }
+
+ .regions {
+     position: absolute;
+     left: 0px;
+     top: 0px;
+     width: 100%;
+     height: 100%;
+
+     .region {
+         position: absolute;
+         z-index: var(--layer-3);
+         cursor: move;
+         border-style: dotted;
+         border-width: 2px;
+
+         .tip {
+             position: absolute;
+             left: 50%;
+             top: 50%;
+             transform: translate(-50%, -50%);
+             font-size: 12px;
+             font-weight: bold;
+             text-align: center;
+             color: red;
+             z-index: var(--layer-2);
+         }
+     }
+ }
+
+ .regions-empty {
+     display: flex;
+     position: relative;
+     height: 10rem;
+     justify-content: center;
+     text-align: center;
+     font-size: 32px;
+     font-weight: bolder;
+     color: var(--comfy-accent-soft);
+
+     span {
+         margin: auto;
+     }
+ }
+</style>

src/lib/widgets/utils.ts

@@ -36,3 +36,39 @@ export function isHidden(widget: IDragItem) {
 
     return false;
 }
+
+export function generateBlankCanvas(width: number, height: number, fill: string = "#fff"): HTMLCanvasElement {
+    const canvas = document.createElement('canvas');
+    canvas.width = width;
+    canvas.height = height;
+    const ctx = canvas.getContext('2d');
+    ctx.save();
+    ctx.fillStyle = fill,
+        ctx.fillRect(0, 0, canvas.width, canvas.height);
+    ctx.restore();
+    return canvas;
+}
+
+export async function loadImage(imageURL: string): Promise<HTMLImageElement> {
+    return new Promise((resolve) => {
+        const e = new Image();
+        e.setAttribute('crossorigin', 'anonymous'); // Don't taint the canvas from loading files on-disk
+        e.addEventListener("load", () => { resolve(e); });
+        e.src = imageURL;
+        return e;
+    });
+}
+
+export async function generateImageCanvas(imageURL: string): Promise<[HTMLCanvasElement, number, number]> {
+    const image = await loadImage(imageURL);
+    const canvas = document.createElement('canvas');
+    canvas.width = image.width;
+    canvas.height = image.height;
+    const ctx = canvas.getContext('2d');
+    ctx.save();
+    ctx.fillStyle = "rgba(255, 255, 255, 0.0)";
+    ctx.fillRect(0, 0, canvas.width, canvas.height);
+    ctx.drawImage(image, 0, 0);
+    ctx.restore();
+    return [canvas, image.width, image.height];
+}