Mask canvas for masked img2img

src/lib/components/ComfyApp.ts

@@ -997,6 +997,7 @@ export default class ComfyApp {
 
                     if ("getPromptThumbnails" in node) {
                         const thumbsToAdd = (node as ComfyGraphNode).getPromptThumbnails();
+                        console.warn("THUMBNAILS", thumbsToAdd)
                         if (thumbsToAdd)
                             thumbnails.push(...thumbsToAdd)
                     }

src/lib/components/ImageUpload.svelte

@@ -3,7 +3,7 @@
  import type { ComfyImageLocation } from "$lib/nodes/ComfyWidgetNodes";
  import notify from "$lib/notify";
  import configState from "$lib/stores/configState";
- import { convertComfyOutputEntryToGradio, convertComfyOutputToComfyURL, type ComfyUploadImageAPIResponse } from "$lib/utils";
+ import { batchUploadFilesToComfyUI, convertComfyOutputToComfyURL, type ComfyBatchUploadResult } from "$lib/utils";
  import { Block, BlockLabel } from "@gradio/atoms";
  import { File as FileIcon } from "@gradio/icons";
  import type { FileData as GradioFileData } from "@gradio/upload";
@@ -59,55 +59,10 @@
      dispatch("image_clicked")
  }
 
- interface GradioUploadResponse {
+ async function upload_files(files: Array<File>): Promise<ComfyBatchUploadResult> {
-     error?: string;
+     console.debug("UPLOADFILES", files);
-     files?: Array<ComfyImageLocation>;
- }
-
- async function upload_files(root: string, files: Array<File>): Promise<GradioUploadResponse> {
-     console.debug("UPLOADILFES", root, files);
-
      dispatch("uploading")
-
+     return batchUploadFilesToComfyUI(files);
-     const url = configState.getBackendURL();
-
-     const requests = files.map(async (file) => {
-         const formData = new FormData();
-         formData.append("image", file, file.name);
-         return fetch(new Request(url + "/upload/image", {
-             body: formData,
-             method: 'POST'
-         }))
-         .then(r => r.json())
-         .catch(error => error);
-     });
-
-     return Promise.all(requests)
-                   .then( (results) => {
-                       const errors = []
-                       const files = []
-
-                       for (const r of results) {
-                           if (r instanceof Error) {
-                               errors.push(r.toString())
-                           }
-                           else {
-                               // bare filename of image
-                               const resp = r as ComfyUploadImageAPIResponse;
-                               files.push({
-                                   filename: resp.name,
-                                   subfolder: "",
-                                   type: "input"
-                               })
-                           }
-                       }
-
-                       let error = null;
-                       if (errors && errors.length > 0)
-                           error = "Upload error(s):\n" + errors.join("\n");
-
-                       return { error, files }
-                   })
  }
 
  $: {
@@ -144,7 +99,7 @@
              );
              let upload_value = _value;
              pending_upload = true;
-             upload_files(root, files).then((response) => {
+             upload_files(files).then((response) => {
                  if (JSON.stringify(upload_value) !== JSON.stringify(_value)) {
                      // value has changed since upload started
                      console.error("[ImageUpload] value has changed since upload started", upload_value, _value)

src/lib/components/MaskCanvas.svelte

@@ -0,0 +1,635 @@
+<script context="module" lang="ts">
+ export type MaskCanvasData = {
+     hasMask: boolean,
+     maskCanvas: HTMLCanvasElement | null,
+ }
+</script>
+
+<script lang="ts">
+ import { loadImage } from "$lib/widgets/utils"
+ import { tick, createEventDispatcher } from "svelte";
+ import { ArrowClockwise, ArrowCounterclockwise, XSquare, Exclude, Circle, Grid3x3Gap, ArrowsFullscreen, FullscreenExit } from "svelte-bootstrap-icons";
+
+ export let fileURL: string | null = null;
+ export let fullscreen: boolean = false;
+
+ const dispatch = createEventDispatcher<{
+     change: MaskCanvasData;
+     release: MaskCanvasData;
+ }>();
+
+ let canvasCursor: string | undefined = undefined;
+
+ let container: HTMLDivElement | null;
+ let canvas: HTMLCanvasElement | null;
+ let maskCanvas: HTMLCanvasElement | null;
+ let renders: HTMLImageElement[] = [];
+ let context: CanvasRenderingContext2D | null;
+ let maskContext: CanvasRenderingContext2D | null;
+ let curLineGroup: LineGroup = [];
+ let redoCurLines: LineGroup = []
+
+ let original: HTMLImageElement | null;
+ let isImageLoaded: boolean = false;
+ let imageWidth: number = 512;
+ let imageHeight: number = 512;
+ let scale: number = 1.0;
+ let minScale: number = 1.0;
+ let brushSize: number = 100;
+ let maskBlur: number = 0;
+ let clipMask: boolean = false;
+ let hasMask: boolean = false;
+
+ let isDrawing: boolean = false;
+ let isPanning: boolean = false;
+ let isBrushShowing: boolean = false;
+
+ let transform = ""
+ $: transform = `translate(${imx}px, ${imy}px) scale(${scale})`
+
+ let imx: number = 0;
+ let imy: number = 0;
+
+ let x: number = 0;
+ let y: number = 0;
+
+ let panX: number = 0;
+ let panY: number = 0;
+
+ const BRUSH_COLOR = "#000"
+
+ enum MouseButton {
+     Left = 0,
+     Middle = 1,
+     Right = 2,
+     Back = 3,
+     Forward = 4
+ }
+
+ $: if (isPanning) {
+     canvasCursor = "grab";
+ }
+ else if (isImageLoaded && isBrushShowing) {
+     canvasCursor = "none";
+ }
+ else {
+     canvasCursor = undefined;
+ }
+
+ $: {
+     context = canvas ? canvas.getContext("2d") : null
+ }
+
+ function clearState() {
+     hasMask = false;
+     maskCanvas = null;
+     maskContext = null;
+     isImageLoaded = false;
+     original = null;
+     renders = []
+     curLineGroup = [];
+     redoCurLines = []
+     imageWidth = 512;
+     imageHeight = 512;
+     scale = 1.0;
+     minScale = 0.5;
+ }
+
+ let loadedFileURL: string | null = null
+ $: if (fileURL !== loadedFileURL) {
+     clearState();
+     if (fileURL) {
+         loadImage(fileURL).then(i => {
+             original = i;
+             isImageLoaded = true;
+         })
+                           .catch(i => {
+                               isImageLoaded = false;
+                           })
+     }
+     else {
+         isImageLoaded = false;
+         original = null;
+     }
+     loadedFileURL = fileURL
+ }
+
+ $: {
+     // initSizeAndScale(isImageLoaded, original);
+     [imageWidth, imageHeight] = getCurrentWidthAndHeight(isImageLoaded, original)
+     scale = initScale(imageWidth, imageHeight)
+     initImagePos()
+ }
+
+ $: hasMask = curLineGroup.length > 0;
+
+ function initImagePos() {
+     if (!container)
+         return
+     const rect = container.getBoundingClientRect();
+     imx = rect.width / 2 - (imageWidth / 2) * scale
+     imy = rect.height / 2 - (imageHeight / 2) * scale
+ }
+
+ function initScale(width: number, height: number): number {
+     const s = getScale(width, height);
+     minScale = s / 2;
+     return s;
+ }
+
+ function initSizeAndScale(isImageLoaded: boolean, original: HTMLImageElement | null) {
+     [imageWidth, imageHeight] = getCurrentWidthAndHeight(isImageLoaded, original)
+     scale = getScale(imageWidth, imageHeight)
+     minScale = scale;
+ }
+
+ type LinePoint = {
+     x: number,
+     y: number
+ }
+ interface Line {
+     size?: number,
+     points: LinePoint[]
+ }
+ type LineGroup = Line[];
+
+ function drawOnCurrentRender(lineGroup: LineGroup) {
+     draw(lineGroup)
+     dispatch("change", {
+         hasMask,
+         maskCanvas
+     })
+ }
+
+ function draw(lineGroup: LineGroup) {
+     if (!context || !maskContext)
+         return
+
+     context.clearRect(0, 0, context.canvas.width, context.canvas.height)
+     maskContext.clearRect(0, 0, maskContext.canvas.width, maskContext.canvas.height)
+
+     const color = BRUSH_COLOR
+
+     const drawMask = (ctx: CanvasRenderingContext2D) => {
+         ctx.save();
+         ctx.filter = `blur(${maskBlur}px)`
+         drawLines(ctx, lineGroup, color)
+         ctx.restore();
+     }
+
+     drawMask(maskContext);
+
+     if (clipMask) {
+         context.save();
+         context.filter = `blur(${maskBlur}px)`
+         drawLines(context, lineGroup, color)
+         context.restore();
+         context.globalCompositeOperation = "source-in"
+         context.drawImage(original!!, 0, 0, imageWidth, imageHeight)
+         context.globalCompositeOperation = "source-over";
+     }
+     else {
+         drawMask(context);
+     }
+ }
+
+ function updateMaskImage() {
+     drawOnCurrentRender(curLineGroup);
+ }
+
+ function drawLines(ctx: CanvasRenderingContext2D, lines: LineGroup, color: string) {
+     ctx.strokeStyle = color
+     ctx.lineCap = 'round'
+     ctx.lineJoin = 'round'
+
+     lines.forEach(line => {
+         if (!line?.points.length || !line.size) {
+             return
+         }
+         ctx.lineWidth = line.size
+         ctx.beginPath()
+         ctx.moveTo(line.points[0].x, line.points[0].y)
+         line.points.forEach(point => ctx.lineTo(point.x, point.y))
+         ctx.stroke()
+     })
+ }
+
+ $: if (canvas && original) {
+     console.warn("INITCANVAS", imageWidth, imageHeight, original.src)
+     maskCanvas = document.createElement("canvas");
+     maskContext = maskCanvas.getContext("2d")!;
+     maskCanvas.width = imageWidth;
+     maskCanvas.height = imageHeight;
+     canvas.width = imageWidth;
+     canvas.height = imageHeight;
+     drawOnCurrentRender([])
+ }
+
+ function getCurrentWidthAndHeight(isImageLoaded: boolean, original: HTMLImageElement | null) {
+     if (isImageLoaded && original){
+         return [original.naturalWidth, original.naturalHeight]
+     }
+     return [512, 512]
+ }
+
+ function getScale(width: number, height: number): number {
+     const size = container?.getBoundingClientRect();
+     if (!size) {
+         return 1.0
+     }
+
+     const ratioWidth = size.width / width
+     const ratioHeight = (size.height) / height
+
+     let scale: number = 1.0
+     if (ratioWidth < 1 || ratioHeight < 1) {
+         scale = Math.min(ratioWidth, ratioHeight)
+     }
+
+     return scale
+ }
+
+ function undoStroke() {
+     if (curLineGroup.length === 0) {
+         return
+     }
+
+     const lastLine = curLineGroup.pop()!
+     const newRedoCurLines = [...redoCurLines, lastLine]
+     redoCurLines = newRedoCurLines
+
+     const newLineGroup = [...curLineGroup]
+     curLineGroup = newLineGroup
+     drawOnCurrentRender(newLineGroup)
+ }
+
+ function redoStroke() {
+     if (redoCurLines.length === 0) {
+         return
+     }
+
+     const line = redoCurLines.pop()!
+     redoCurLines = [...redoCurLines]
+
+     const newLineGroup = [...curLineGroup, line]
+     curLineGroup = newLineGroup
+     drawOnCurrentRender(newLineGroup)
+ }
+
+ export function clearStrokes() {
+     redoCurLines = []
+     const newLineGroup: LineGroup = []
+     curLineGroup = newLineGroup
+     drawOnCurrentRender(newLineGroup)
+ }
+
+ export function recenterImage() {
+     scale = initScale(imageWidth, imageHeight)
+     initImagePos();
+ }
+
+ async function toggleFullscreen() {
+     fullscreen = !fullscreen;
+     updateMaskImage();
+     await tick();
+     recenterImage();
+ }
+
+ function onCanvasMouseOver() {
+     isBrushShowing = true;
+ }
+
+ function onCanvasFocus() {
+     isBrushShowing = true;
+ }
+
+ function onCanvasMouseLeave() {
+     isBrushShowing = false;
+ }
+
+ function mouseXY(e: MouseEvent): LinePoint {
+     return { x: e.offsetX, y: e.offsetY }
+ }
+
+ function onCanvasMouseDown(e: MouseEvent) {
+     if (!original?.src)
+         return;
+
+     if (isPanning)
+         return;
+
+     if (canvas == null)
+         return;
+
+     switch (e.button) {
+         case MouseButton.Right:
+             return;
+         case MouseButton.Middle:
+             isPanning = true;
+             panX = e.offsetX * scale;
+             panY = e.offsetY * scale;
+             return;
+     }
+
+     isDrawing = true;
+
+     redoCurLines = []
+
+     let lineGroup: LineGroup = [...curLineGroup]
+     lineGroup.push({size: brushSize, points: [mouseXY(e)] })
+     curLineGroup = lineGroup
+     drawOnCurrentRender(curLineGroup);
+ }
+
+ function onCanvasMouseUp() {
+
+ }
+
+ function onCanvasMouseMove(e: MouseEvent) {
+     if (isPanning)
+         return;
+
+     if (!isDrawing)
+         return;
+
+     if (curLineGroup.length === 0)
+         return
+
+     curLineGroup[curLineGroup.length-1].points.push(mouseXY(e))
+     curLineGroup = curLineGroup; // react
+     drawOnCurrentRender(curLineGroup)
+ }
+
+ function onCanvasMouseWheel(e: WheelEvent) {
+     e.preventDefault();
+     if (!container || e.target != canvas)
+         return;
+
+     const bound = container.getBoundingClientRect()
+
+     // coodinates on the image that were zoomed
+     const x_ = e.clientX - bound.x
+     const y_ = e.clientY - bound.y
+
+     e.preventDefault();
+     var delta = e.deltaY * -0.001
+     delta = Math.max(-1,Math.min(1,delta)) // cap the delta to [-1,1] for cross browser consistency
+
+     const zx = (x_ - imx)/scale
+     const zy = (y_ - imy)/scale
+
+     scale += delta * scale
+     scale = Math.max(minScale,scale)
+
+     imx = -zx * scale + x_
+     imy = -zy * scale + y_
+
+     x = e.offsetX * scale;
+     y = e.offsetY * scale;
+ }
+
+ function onMouseMove(e: MouseEvent) {
+     if (e.target != canvas)
+         return;
+
+     x = e.offsetX * scale;
+     y = e.offsetY * scale;
+
+     if (isPanning) {
+         imx += x - panX;
+         imy += y - panY;
+     }
+ }
+
+ function onMouseUp(e: MouseEvent) {
+     if (e.button === MouseButton.Middle) {
+         isPanning = false
+         panX = 0
+         panY = 0
+     }
+
+     if (isPanning)
+         return;
+
+     if (!original?.src)
+         return;
+
+     if (!canvas)
+         return;
+
+     if (!isDrawing)
+         return;
+
+     isDrawing = false;
+     dispatch("release", { hasMask, maskCanvas })
+ }
+
+ function dispatchRelease() {
+     dispatch("release", { hasMask, maskCanvas })
+ }
+</script>
+
+<svelte:window on:mouseup={onMouseUp} />
+
+<div class="me-container" class:fullscreen bind:this={container} on:mousemove={onMouseMove}>
+    {#if !isImageLoaded}
+        <div>
+            (empty)
+        </div>
+    {:else}
+        <div class="me-transform" style:transform={transform} style:--scale={scale}>
+            <div class="me-canvas-container">
+                <div class="me-original-image-container"
+                     style:width="{imageWidth}px"
+                     style:height="{imageHeight}px">
+                    {#if original}
+                        {@const showOriginal = !clipMask}
+                        <img class="me-original-image"
+                             src={original.src}
+                             style:width={imageWidth}
+                             style:height={imageHeight}
+                             style:display={showOriginal ? "block" : "none"}
+                        />
+                    {/if}
+                </div>
+                <canvas class="me-canvas"
+                        bind:this={canvas}
+                        style:cursor={canvasCursor}
+                        on:mouseover={onCanvasMouseOver}
+                        on:focus={onCanvasFocus}
+                        on:wheel={onCanvasMouseWheel}
+                        on:mouseleave={onCanvasMouseLeave}
+                        on:mousedown|preventDefault={onCanvasMouseDown}
+                        on:mouseup|preventDefault={onCanvasMouseUp}
+                        on:mousemove={onCanvasMouseMove}
+                />
+            </div>
+        </div>
+    {/if}
+    {#if isImageLoaded && isBrushShowing && !isPanning}
+        <div class="me-brush-cursor"
+             style:width="{brushSize * scale}px"
+             style:height="{brushSize * scale}px"
+             style:left="{x + imx}px"
+             style:top="{y + imy}px"
+             style:transform="translate(-50%, -50%)"
+        />
+    {/if}
+    <div class="me-toolkit-bar">
+        <button disabled={curLineGroup.length === 0} on:click={undoStroke}>
+            <ArrowCounterclockwise />
+        </button>
+        <button disabled={redoCurLines.length === 0} on:click={redoStroke}>
+            <ArrowClockwise />
+        </button>
+        <button on:click={clearStrokes} disabled={curLineGroup.length === 0 && redoCurLines.length === 0}>
+            <XSquare/>
+        </button>
+        <label>
+            <Circle />
+            <input type="range" min="1" max="200" bind:value={brushSize} step="0.1"
+                   on:change={updateMaskImage}
+                   on:pointerup={dispatchRelease}/>
+        </label>
+        <label>
+            <Grid3x3Gap/>
+            <input type="range" min="1" max="100" bind:value={maskBlur} step="0.1"
+                   on:change={updateMaskImage}
+                   on:pointerup={dispatchRelease}/>
+        </label>
+        <div class="toggle-button" class:toggled={clipMask} on:click={() => {clipMask = !clipMask; updateMaskImage()}}>
+            <Exclude />
+        </div>
+        <div class="toggle-button" class:toggled={fullscreen} on:click={() => {toggleFullscreen()}}>
+            {#if fullscreen}
+                <FullscreenExit />
+            {:else}
+                <ArrowsFullscreen />
+            {/if}
+        </div>
+    </div>
+</div>
+
+<style lang="scss">
+ $bg-color: #a0a0a0;
+ .me-container {
+     width: 100%;
+     height: 100%;
+     position: relative;
+     overflow: hidden;
+     background-color: white;
+     background-image:
+         linear-gradient(45deg, #ccc 25%, transparent 25%),
+         linear-gradient(135deg, #ccc 25%, transparent 25%),
+         linear-gradient(45deg, transparent 75%, #ccc 75%),
+         linear-gradient(135deg, transparent 75%, #ccc 75%);
+     background-size:25px 25px; /* Must be a square */
+     background-position:0 0, 12.5px 0, 12.5px -12.5px, 0px 12.5px; /* Must be half of one side of the square */
+
+     &.fullscreen {
+         position: fixed;
+         top: 0;
+         left: 0;
+         width: 100vw;
+         height: 100vh;
+         margin: auto;
+         z-index: var(--layer-top);
+     }
+ }
+
+ .me-transform {
+     --scale: 1;
+     display: flex;
+     flex-wrap: wrap;
+     width: -moz-fit-content;
+     width: fit-content;
+     height: -moz-fit-content;
+     height: fit-content;
+     margin: 0;
+     padding: 0;
+     transform-origin: 0% 0%;
+ }
+
+ .me-original-image-container {
+     position: absolute;
+     top: 0;
+     left: 0;
+     grid-area: editor-content;
+     pointer-events: none;
+     user-select: none;
+     display: grid;
+     grid-template-areas: 'original-image-content';
+     border: calc((1 / var(--scale)) * 5px) dashed grey;
+
+     img.me-original-image {
+         grid-area: original-image-content;
+     }
+ }
+
+ .me-canvas {
+     position: absolute;
+     z-index: 1;
+ }
+
+ .me-brush-cursor {
+     position: absolute;
+     border-radius: 50%;
+     background-color: #000;
+     border: 1px solid var(--yellow-accent);
+     pointer-events: none;
+ }
+
+ .me-toolkit-bar {
+     position: absolute;
+     bottom: 0.5rem;
+     border-radius: 3rem;
+     padding: 0.4rem 24px;
+     display: flex;
+     margin: 0.5rem auto;
+     gap: 16px;
+     left: 0;
+     right: 0;
+     width: 80%;
+     height: 3rem;
+     align-items: center;
+     justify-content: space-evenly;
+     backdrop-filter: blur(12px);
+     background-color: white;
+     animation: slideUp 0.2s ease-out;
+     border: var(--editor-toolkit-panel-border);
+     box-shadow: 0 0 0 4px #0000001a, 0 3px 16px #00000014, 0 2px 6px 1px #00000017;
+
+     label {
+         display: flex;
+         flex-direction: row;
+         gap: 4px;
+         input {
+             width: 5rem;
+         }
+     }
+
+     button {
+         &:not(:disabled) {
+             cursor: pointer;
+         }
+         &:hover:not(:disabled) {
+             color: var(--secondary-600);
+         }
+         &:disabled {
+             opacity: 40%;
+         }
+     }
+
+     .toggle-button {
+         &:hover:not(:disabled) {
+             color: var(--secondary-600);
+         }
+         &:not(:disabled) {
+             cursor: pointer;
+         }
+         &.toggled {
+             color: var(--secondary-400);
+         }
+     }
+ }
+</style>

src/lib/nodes/ComfyPickImageNode.ts

@@ -1,8 +1,17 @@
 import { LiteGraph, type ITextWidget, type SlotLayout, type INumberWidget } from "@litegraph-ts/core";
-import ComfyGraphNode from "./ComfyGraphNode";
+import ComfyGraphNode, { type ComfyGraphNodeProperties } from "./ComfyGraphNode";
 import { comfyFileToAnnotatedFilepath, type ComfyBoxImageMetadata } from "$lib/utils";
 
+export interface ComfyPickImageProperties extends ComfyGraphNodeProperties {
+    imageTagFilter: string
+}
+
 export default class ComfyPickImageNode extends ComfyGraphNode {
+    override properties: ComfyPickImageProperties = {
+        tags: [],
+        imageTagFilter: ""
+    }
+
     static slotLayout: SlotLayout = {
         inputs: [
             { name: "images", type: "COMFYBOX_IMAGES,COMFYBOX_IMAGE" },
@@ -13,57 +22,87 @@ export default class ComfyPickImageNode extends ComfyGraphNode {
             { name: "filename", type: "string" },
             { name: "width", type: "number" },
             { name: "height", type: "number" },
+            { name: "children", type: "COMFYBOX_IMAGES" },
         ]
     }
 
+    tagFilterWidget: ITextWidget;
     filepathWidget: ITextWidget;
     folderWidget: ITextWidget;
     widthWidget: INumberWidget;
     heightWidget: INumberWidget;
+    tagsWidget: ITextWidget;
+    childrenWidget: INumberWidget;
 
     constructor(title?: string) {
         super(title)
+        this.tagFilterWidget = this.addWidget("text", "Tag Filter", this.properties.imageTagFilter, "imageTagFilter")
+
         this.filepathWidget = this.addWidget("text", "File", "")
+        this.filepathWidget.disabled = true;
         this.folderWidget = this.addWidget("text", "Folder", "")
+        this.folderWidget.disabled = true;
         this.widthWidget = this.addWidget("number", "Width", 0)
+        this.widthWidget.disabled = true;
         this.heightWidget = this.addWidget("number", "Height", 0)
-        for (const widget of this.widgets)
+        this.heightWidget.disabled = true;
-            widget.disabled = true;
+        this.tagsWidget = this.addWidget("text", "Tags", "")
+        this.tagsWidget.disabled = true;
+        this.childrenWidget = this.addWidget("number", "# of Children", 0)
+        this.childrenWidget.disabled = true;
     }
 
     _value: ComfyBoxImageMetadata[] | null = null;
     _image: ComfyBoxImageMetadata | null = null;
     _path: string | null = null;
-    _index: number = 0;
+    _index: number | null = null;
 
     private setValue(value: ComfyBoxImageMetadata[] | ComfyBoxImageMetadata | null, index: number) {
         if (value != null && !Array.isArray(value)) {
             value = [value]
             index = 0;
         }
-        const changed = this._value != value || this._index != index;
         this._value = value as ComfyBoxImageMetadata[];
         this._index = index;
+        let image: ComfyBoxImageMetadata | null = null;
+        if (value && this._index != null && value[this._index] != null) {
+            image = value[this._index];
+        }
+
+        const changed = this._value != value || this._index != index || this._image != image;
+
         if (changed) {
-            if (value && value[this._index] != null) {
+            if (image) {
-                this._image = value[this._index]
+                this._image = image
+                this._image.children ||= []
+                this._image.tags ||= []
+
                 this._path = comfyFileToAnnotatedFilepath(this._image.comfyUIFile);
                 this.filepathWidget.value = this._image.comfyUIFile.filename
                 this.folderWidget.value = this._image.comfyUIFile.type
+                this.childrenWidget.value = this._image.children.length
+                this.tagsWidget.value = this._image.tags.join(", ")
             }
             else {
                 this._image = null;
                 this._path = null;
                 this.filepathWidget.value = "(None)"
                 this.folderWidget.value = ""
+                this.childrenWidget.value = 0
+                this.tagsWidget.value = ""
             }
-            console.log("SET", value, this._image, this._path)
+
+            console.log("SET", value, this._image, this._path, this.properties.imageTagFilter)
         }
     }
 
     override onExecute() {
         const data = this.getInputData(0)
-        const index = this.getInputData(1) || 0
+        let index = this.getInputData(1);
+        if (this.properties.imageTagFilter && Array.isArray(data))
+            index = data.findIndex(i => i.tags?.includes(this.properties.imageTagFilter))
+        else
+            index = 0;
         this.setValue(data, index);
 
         if (this._image == null) {
@@ -71,6 +110,7 @@ export default class ComfyPickImageNode extends ComfyGraphNode {
             this.setOutputData(1, null)
             this.setOutputData(2, 0)
             this.setOutputData(3, 0)
+            this.setOutputData(4, null)
 
             this.widthWidget.value = 0
             this.heightWidget.value = 0
@@ -80,6 +120,7 @@ export default class ComfyPickImageNode extends ComfyGraphNode {
             this.setOutputData(1, this._path);
             this.setOutputData(2, this._image.width);
             this.setOutputData(3, this._image.height);
+            this.setOutputData(4, this._image.children);
 
             // XXX: image size doesn't load until the <img> element is ready on
             // the page so this can come after several frames' worth of

src/lib/nodes/widgets/ComfyImageUploadNode.ts

@@ -8,12 +8,14 @@ import ComfyWidgetNode from "./ComfyWidgetNode";
 import { get, writable, type Writable } from "svelte/store";
 
 export interface ComfyImageUploadNodeProperties extends ComfyWidgetProperties {
+    maskCount: number
 }
 
 export default class ComfyImageUploadNode extends ComfyWidgetNode<ComfyBoxImageMetadata[]> {
     properties: ComfyImageUploadNodeProperties = {
         defaultValue: [],
         tags: [],
+        maskCount: 0
     }
 
     static slotLayout: SlotLayout = {
@@ -39,13 +41,21 @@ export default class ComfyImageUploadNode extends ComfyWidgetNode<ComfyBoxImageM
         super(name, [])
     }
 
-    override onExecute() {
+    override onExecute(param: any, options: object) {
         // TODO better way of getting image size?
         const value = get(this.value)
         if (value && value.length > 0) {
             value[0].width = get(this.imgWidth)
             value[0].height = get(this.imgHeight)
+
+            // NOTE: assumes masks will have the same image size as the parent image!
+            for (const child of value[0].children) {
+                child.width = get(this.imgWidth)
+                child.height = get(this.imgHeight)
+            }
         }
+
+        super.onExecute(param, options);
     }
 
     override parseValue(value: any): ComfyBoxImageMetadata[] {

src/lib/stores/layoutStates.ts

@@ -616,6 +616,19 @@ const ALL_ATTRIBUTES: AttributesSpecList = [
                 defaultValue: true
             },
 
+            // ImageUpload
+            {
+                name: "maskCount",
+                type: "number",
+                location: "nodeProps",
+                editable: true,
+                validNodeTypes: ["ui/image_upload"],
+                defaultValue: 0,
+                min: 0,
+                max: 8,
+                step: 1
+            },
+
             // Radio
             {
                 name: "defaultValue",

src/lib/utils.ts

@@ -75,6 +75,13 @@ export function download(filename: string, text: string, type: string = "text/pl
     }, 0);
 }
 
+export function downloadCanvas(canvas: HTMLCanvasElement, filename: string, type: string = "image/png") {
+    var link = document.createElement('a');
+    link.download = filename;
+    link.href = canvas.toDataURL(type);
+    link.click();
+}
+
 export const MAX_LOCAL_STORAGE_MB = 5;
 
 export function getLocalStorageUsedMB(): number {
@@ -434,6 +441,8 @@ export type ComfyBoxImageMetadata = {
     width?: number,
     /* Image height. */
     height?: number,
+    /* Child images associated with this image, like masks. */
+    children: ComfyBoxImageMetadata[]
 }
 
 export function isComfyBoxImageMetadata(value: any): value is ComfyBoxImageMetadata {
@@ -458,6 +467,7 @@ export function filenameToComfyBoxMetadata(filename: string, type: ComfyUploadIm
         },
         name: "Filename",
         tags: [],
+        children: []
     }
 }
 
@@ -467,6 +477,7 @@ export function comfyFileToComfyBoxMetadata(comfyUIFile: ComfyImageLocation): Co
         comfyUIFile,
         name: "File",
         tags: [],
+        children: []
     }
 }
 
@@ -628,3 +639,70 @@ export function playSound(sound: string) {
     const audio = new Audio(url);
     audio.play();
 }
+
+export interface ComfyBatchUploadResult {
+    error?: string;
+    files: Array<ComfyImageLocation>;
+}
+
+export type ComfyBatchBlob = {
+    blob: Blob,
+    filename: string,
+    overwrite?: boolean
+}
+
+export async function batchUploadFilesToComfyUI(files: Array<File>): Promise<ComfyBatchUploadResult> {
+    const blobs = files.map(f => { return { blob: f, filename: f.name } })
+    return batchUploadBlobsToComfyUI(blobs)
+}
+
+export async function batchUploadBlobsToComfyUI(blobs: ComfyBatchBlob[]): Promise<ComfyBatchUploadResult> {
+    const url = configState.getBackendURL();
+
+    const requests = blobs.map(async (blob) => {
+        const formData = new FormData();
+        formData.append("image", blob.blob, blob.filename);
+        if (blob.overwrite) {
+            formData.append("overwrite", "true")
+        }
+        return fetch(new Request(url + "/upload/image", {
+            body: formData,
+            method: 'POST'
+        }))
+            .then(r => r.json())
+            .catch(error => error);
+    });
+
+    return Promise.all(requests)
+        .then((results) => {
+            const errors = []
+            const files = []
+
+            for (const r of results) {
+                if (r instanceof Error) {
+                    errors.push(r.toString())
+                }
+                else {
+                    // bare filename of image
+                    const resp = r as ComfyUploadImageAPIResponse;
+                    files.push({
+                        filename: resp.name,
+                        subfolder: "",
+                        type: "input"
+                    })
+                }
+            }
+
+            let error = null;
+            if (errors && errors.length > 0)
+                error = "Upload error(s):\n" + errors.join("\n");
+
+            return { error, files }
+        })
+}
+
+export function canvasToBlob(canvas: HTMLCanvasElement): Promise<Blob> {
+    return new Promise(function(resolve) {
+        canvas.toBlob(resolve);
+    });
+}

src/lib/widgets/ImageUploadWidget.svelte

@@ -3,46 +3,143 @@
  import { Block } from "@gradio/atoms";
  import { TextBox } from "@gradio/form";
  import Row from "$lib/components/gradio/app/Row.svelte";
- import { get, writable, type Writable } from "svelte/store";
+ import { writable, type Writable } from "svelte/store";
- import Modal from "$lib/components/Modal.svelte";
  import { Button } from "@gradio/button";
- import { type Embed as Klecks } from "klecks";
 
- import "klecks/style/style.scss";
  import ImageUpload from "$lib/components/ImageUpload.svelte";
- import { uploadImageToComfyUI, type ComfyBoxImageMetadata, comfyFileToComfyBoxMetadata, comfyBoxImageToComfyURL, comfyBoxImageToComfyFile, type ComfyUploadImageType, type ComfyImageLocation } from "$lib/utils";
+ import {
- import configState from "$lib/stores/configState";
+     type ComfyBoxImageMetadata,
+     comfyFileToComfyBoxMetadata,
+     comfyBoxImageToComfyFile,
+     type ComfyImageLocation,
+     comfyBoxImageToComfyURL,
+     convertComfyOutputToComfyURL,
+     batchUploadBlobsToComfyUI,
+     canvasToBlob,
+
+	 basename
+
+ } from "$lib/utils";
  import notify from "$lib/notify";
- import NumberInput from "$lib/components/NumberInput.svelte";
+ import { ImageViewer } from "$lib/ImageViewer";
-	import type { ComfyImageEditorNode } from "$lib/nodes/widgets";
+ import MaskCanvas, { type MaskCanvasData } from "$lib/components/MaskCanvas.svelte";
-	import { ImageViewer } from "$lib/ImageViewer";
+ import type { ComfyImageUploadNode } from "$lib/nodes/widgets";
-	import { generateBlankCanvas, generateImageCanvas } from "./utils";
+	import { tick } from "svelte";
 
  export let widget: WidgetLayout | null = null;
  export let isMobile: boolean = false;
- let node: ComfyImageEditorNode | null = null;
+ let node: ComfyImageUploadNode | null = null;
  let nodeValue: Writable<ComfyBoxImageMetadata[]> | null = null;
- let attrsChanged: Writable<number> | null = null;
 
  let imgWidth: Writable<number> = writable(0);
  let imgHeight: Writable<number> = writable(0);
 
+ let maskCanvasComp: MaskCanvas | null = null;
+ let editMask: boolean = false;
+
  $: widget && setNodeValue(widget);
 
+ let canMask = false;
+ $: canMask = (node?.properties?.maskCount || 0) > 0;
+ $: if (!canMask) clearMask();
+
+
  function setNodeValue(widget: WidgetLayout) {
      if (widget) {
-         node = widget.node as ComfyImageEditorNode
+         node = widget.node as ComfyImageUploadNode
          nodeValue = node.value;
-         attrsChanged = widget.attrsChanged;
          imgWidth = node.imgWidth
          imgHeight = node.imgHeight
          status = $nodeValue && $nodeValue.length > 0 ? "uploaded" : "empty"
      }
  };
 
+ let hasImage = false;
+
+ $: hasImage = $nodeValue && $nodeValue.length > 0;
+ $: if (!hasImage) {
+     editMask = false;
+ }
+
+ const MASK_FILENAME: string = "ComfyBoxMask.png"
+
+ async function onMaskReleased(e: CustomEvent<MaskCanvasData>) {
+     const data = e.detail;
+     if (data.maskCanvas && data.hasMask) {
+         await saveMask(data.maskCanvas)
+     }
+ }
+
+ async function saveMask(maskCanvas: HTMLCanvasElement) {
+     if (!canMask) {
+         notify("Mask editing is disabled for this widget.", { type: "warning" })
+         return;
+     }
+     if (!maskCanvas) {
+         notify("No mask canvas!", { type: "warning" })
+         return
+     }
+     if (!$nodeValue || $nodeValue.length === 0) {
+         notify("No image uploaded to apply mask to.", { type: "warning" })
+         return
+     }
+
+     const hadNoMask = $nodeValue[0].children.findIndex(i => i.tags?.includes("mask")) === -1;
+     const existFilename = $nodeValue[0].comfyUIFile.filename
+     const filename = existFilename ? `${basename(existFilename)}_mask.png` : MASK_FILENAME
+
+     console.warn("[ImageUpload] UPLOAD MASK", filename)
+
+     await canvasToBlob(maskCanvas)
+         .then(blob => batchUploadBlobsToComfyUI([{
+             blob,
+             filename,
+             overwrite: true
+         }]))
+         .then(result => {
+             const meta = result.files.map(f => {
+                 const m = comfyFileToComfyBoxMetadata(f)
+                 m.tags = ["mask"]
+                 m.width = maskCanvas.width;
+                 m.height = maskCanvas.height;
+                 return m;
+             });
+             if ($nodeValue.length > 0) {
+                 // TODO support multiple images?
+                 $nodeValue[0].children = meta;
+                 if (hadNoMask) {
+                     notify("Uploaded mask successfully!", { type: "success" })
+                 }
+             }
+             else {
+                 throw new Error("No image was uploaded yet.")
+             }
+         })
+         .catch(error => {
+             notify(`Failed to upload mask to ComfyUI: ${error}`, { type: "error", timeout: 10000 })
+         })
+ }
+
+ function clearMask() {
+     for (const image of $nodeValue) {
+         // TODO other child image types preserved here?
+         image.children = [];
+     }
+     if (maskCanvasComp) {
+         maskCanvasComp.clearStrokes();
+     }
+ }
+
+ async function toggleEditMask() {
+     editMask = !editMask;
+     await tick();
+     if (maskCanvasComp) {
+         maskCanvasComp.recenterImage();
+     }
+ }
+
  let editorRoot: HTMLDivElement | null = null;
  let showModal = false;
- let kl: Klecks | null = null;
 
  function disposeEditor() {
      console.warn("[ImageEditorWidget] CLOSING", widget, $nodeValue)
@@ -53,100 +150,9 @@
          }
      }
 
-     kl = null;
      showModal = false;
  }
 
- const FILENAME: string = "ComfyUITemp.png";
- const SUBFOLDER: string = "ComfyBox_Editor";
- const DIRECTORY: ComfyUploadImageType = "input";
-
- async function submitKlecksToComfyUI(onSuccess: () => void, onError: () => void) {
-     const blob = kl.getPNG();
-
-     status = "uploading"
-
-     await uploadImageToComfyUI(blob, FILENAME, DIRECTORY, SUBFOLDER)
-         .then((entry: ComfyImageLocation) => {
-             const meta: ComfyBoxImageMetadata = comfyFileToComfyBoxMetadata(entry);
-             $nodeValue = [meta] // TODO more than one image
-             status = "uploaded"
-             notify("Saved image to ComfyUI!", { type: "success" })
-             onSuccess();
-         })
-         .catch(err => {
-             notify(`Failed to upload image from editor: ${err}`, { type: "error", timeout: 10000 })
-             status = "error"
-             uploadError = err;
-             $nodeValue = []
-             onError();
-         })
- }
-
- let closeDialog = null;
-
- async function saveAndClose() {
-     console.log(closeDialog, kl)
-     if (!closeDialog || !kl)
-         return;
-
-     submitKlecksToComfyUI(() => {}, () => {});
-     closeDialog()
- }
-
- let blankImageWidth = 512;
- let blankImageHeight = 512;
-
- let klecks: typeof import("klecks") | null = null;
-
- async function openImageEditor() {
-     if (!editorRoot)
-         return;
-
-     showModal = true;
-
-     const url = configState.getBackendURL();
-
-     klecks ||= await import("klecks");
-
-     kl = new klecks.Embed({
-         embedUrl: url,
-         onSubmit: submitKlecksToComfyUI,
-         targetEl: editorRoot,
-         warnOnPageClose: false
-     });
-
-     console.warn("[ImageEditorWidget] OPENING", widget, $nodeValue)
-
-     let canvas = null;
-     let width = blankImageWidth;
-     let height = blankImageHeight;
-
-     if ($nodeValue && $nodeValue.length > 0) {
-         const comfyImage = $nodeValue[0];
-         const comfyURL = comfyBoxImageToComfyURL(comfyImage);
-         [canvas, width, height] = await generateImageCanvas(comfyURL);
-     }
-     else {
-         canvas = generateBlankCanvas(width, height);
-     }
-
-     kl.openProject({
-         width: width,
-         height: height,
-         layers: [{
-             name: 'Image',
-             opacity: 1,
-             mixModeStr: 'source-over',
-             image: canvas
-         }]
-     });
-
-     setTimeout(function () {
-         kl?.klApp?.out("yo");
-     }, 1000);
- }
-
  function openLightbox() {
      if (!$nodeValue || $nodeValue.length === 0)
          return;
@@ -189,9 +195,6 @@
      notify(`Failed to upload image to ComfyUI: ${uploadError}`, { type: "error", timeout: 10000 })
  }
 
- function onChange(e: CustomEvent<ComfyImageLocation[]>) {
- }
-
  let _value: ComfyImageLocation[] = []
  $: if ($nodeValue)
      _value = $nodeValue.map(comfyBoxImageToComfyFile)
@@ -199,6 +202,10 @@
      _value = []
 
  $: canEdit = status === "empty" || status === "uploaded";
+
+
+ function onChange(e: CustomEvent<ComfyImageLocation[]>) {
+ }
 </script>
 
 <div class="wrapper comfy-image-editor">
@@ -219,64 +226,50 @@
         />
     {:else}
         <div class="comfy-image-editor-panel">
-            <ImageUpload value={_value}
+            {#if _value && canMask}
-                         bind:imgWidth={$imgWidth}
+                {@const comfyURL = convertComfyOutputToComfyURL(_value[0])}
-                         bind:imgHeight={$imgHeight}
+                <div class="mask-canvas-wrapper" style:display={editMask ? "block" : "none"}>
-                         fileCount={"single"}
+                    <MaskCanvas bind:this={maskCanvasComp} fileURL={comfyURL} on:release={onMaskReleased} />
-                         elem_classes={[]}
-                         style={""}
-                         label={widget.attrs.title}
-                         on:uploading={onUploading}
-                         on:uploaded={onUploaded}
-                         on:upload_error={onUploadError}
-                         on:clear={onClear}
-                         on:change={onChange}
-                         on:image_clicked={openLightbox}
-            />
-            <Modal bind:showModal closeOnClick={false} on:close={disposeEditor} bind:closeDialog>
-                <div>
-                    <div id="klecks-loading-screen">
-                        <span id="klecks-loading-screen-text"></span>
-                    </div>
-                    <div class="image-editor-root" bind:this={editorRoot} />
                 </div>
-                <div slot="buttons">
+            {/if}
-                    <Button variant="primary" on:click={saveAndClose}>
+            <div style:display={(canMask && editMask) ? "none" : "block"}>
-                        Save and Close
+                <ImageUpload value={_value}
-                    </Button>
+                             bind:imgWidth={$imgWidth}
-                    <Button variant="secondary" on:click={closeDialog}>
+                             bind:imgHeight={$imgHeight}
-                        Discard Edits
+                             fileCount={"single"}
-                    </Button>
+                             elem_classes={[]}
-                </div>
+                             style={""}
-            </Modal>
+                             label={widget.attrs.title}
+                             on:uploading={onUploading}
+                             on:uploaded={onUploaded}
+                             on:upload_error={onUploadError}
+                             on:clear={onClear}
+                             on:change={onChange}
+                             on:image_clicked={openLightbox}
+                />
+            </div>
             <Block>
-                {#if !$nodeValue || $nodeValue.length === 0}
+                {#if hasImage}
+                    {@const maskCount = $nodeValue[0] ? $nodeValue[0].children.filter(f => f.tags?.includes("mask")).length : 0}
                     <Row>
-                        <Row>
+                        {#if canMask}
-                            <Button variant="secondary" disabled={!canEdit} on:click={openImageEditor}>
-                                Create Image
-                            </Button>
                             <div>
-                                <TextBox show_label={false} disabled={true} value="Status: {status}"/>
+                                {#if editMask}
+                                    <Button variant="secondary" on:click={() => { clearMask(); notify("Mask cleared."); }}>
+                                        Clear Mask
+                                    </Button>
+                                {/if}
+                                <Button disabled={!_value} on:click={toggleEditMask}>
+                                    {#if editMask}
+                                        Show Image
+                                    {:else}
+                                        Edit Mask
+                                    {/if}
+                                </Button>
                             </div>
-                            {#if uploadError}
+                        {/if}
-                                <div>
-                                    Upload error: {uploadError}
-                                </div>
-                            {/if}
-                        </Row>
-                        <Row>
-                            <NumberInput label={"Width"} min={64} max={2048} step={64} bind:value={blankImageWidth} />
-                            <NumberInput label={"Height"} min={64} max={2048} step={64} bind:value={blankImageHeight} />
-                        </Row>
-                    </Row>
-                {:else}
-                    <Row>
-                        <Button variant="secondary" disabled={!canEdit} on:click={openImageEditor}>
-                            Edit Image
-                        </Button>
                         <div>
-                            <TextBox label={""} show_label={false} disabled={true} lines={1} max_lines={1} value="Status: {status}"/>
+                            <TextBox label={""} show_label={false} disabled={true} lines={1} max_lines={1} value="Images: {$nodeValue.length}, masks: {maskCount}"/>
                         </div>
                         {#if uploadError}
                             <div>
@@ -291,25 +284,13 @@
 </div>
 
 <style lang="scss">
- .image-editor-root {
-     width: 75vw;
-     height: 75vh;
-     overflow: hidden;
-
-     color: black;
-
-     :global(> .g-root) {
-         height: calc(100% - 59px);
-     }
- }
-
  .comfy-image-editor {
      :global(> dialog) {
          overflow: hidden;
      }
  }
 
- :global(.kl-popup) {
+ .mask-canvas-wrapper {
-     z-index: 999999999999;
+     height: calc(var(--size-96) * 1.5);
  }
 </style>