neuro_llc/ComfyBox
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 1 Wiki Activity

Mask canvas for masked img2img

Browse Source
This commit is contained in:
space-nuko
2023-05-27 20:37:22 -05:00
parent 7e2b6111dd
commit cb9e8540a0
9 changed files with 5415 additions and 235 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
src/lib/components/ComfyApp.ts
View File

@@ -997,6 +997,7 @@ export default class ComfyApp {
if ("getPromptThumbnails" in node) {
const thumbsToAdd = (node as ComfyGraphNode).getPromptThumbnails();
console.warn("THUMBNAILS", thumbsToAdd)
if (thumbsToAdd)
thumbnails.push(...thumbsToAdd)
}

55
src/lib/components/ImageUpload.svelte
View File

@@ -3,7 +3,7 @@
import type { ComfyImageLocation } from "$lib/nodes/ComfyWidgetNodes";
import notify from "$lib/notify";
import configState from "$lib/stores/configState";
import { convertComfyOutputEntryToGradio, convertComfyOutputToComfyURL, type ComfyUploadImageAPIResponse } from "$lib/utils";
import { batchUploadFilesToComfyUI, convertComfyOutputToComfyURL, type ComfyBatchUploadResult } from "$lib/utils";
import { Block, BlockLabel } from "@gradio/atoms";
import { File as FileIcon } from "@gradio/icons";
import type { FileData as GradioFileData } from "@gradio/upload";
@@ -59,55 +59,10 @@
dispatch("image_clicked")
}
interface GradioUploadResponse {
error?: string;
files?: Array<ComfyImageLocation>;
}
async function upload_files(root: string, files: Array<File>): Promise<GradioUploadResponse> {
console.debug("UPLOADILFES", root, files);
async function upload_files(files: Array<File>): Promise<ComfyBatchUploadResult> {
console.debug("UPLOADFILES", files);
dispatch("uploading")
const url = configState.getBackendURL();
const requests = files.map(async (file) => {
const formData = new FormData();
formData.append("image", file, file.name);
return fetch(new Request(url + "/upload/image", {
body: formData,
method: 'POST'
}))
.then(r => r.json())
.catch(error => error);
});
return Promise.all(requests)
.then( (results) => {
const errors = []
const files = []
for (const r of results) {
if (r instanceof Error) {
errors.push(r.toString())
}
else {
// bare filename of image
const resp = r as ComfyUploadImageAPIResponse;
files.push({
filename: resp.name,
subfolder: "",
type: "input"
})
}
}
let error = null;
if (errors && errors.length > 0)
error = "Upload error(s):\n" + errors.join("\n");
return { error, files }
})
return batchUploadFilesToComfyUI(files);
}
$: {
@@ -144,7 +99,7 @@
);
let upload_value = _value;
pending_upload = true;
upload_files(root, files).then((response) => {
upload_files(files).then((response) => {
if (JSON.stringify(upload_value) !== JSON.stringify(_value)) {
// value has changed since upload started
console.error("[ImageUpload] value has changed since upload started", upload_value, _value)

635
src/lib/components/MaskCanvas.svelte Normal file
View File

@@ -0,0 +1,635 @@
<script context="module" lang="ts">
export type MaskCanvasData = {
hasMask: boolean,
maskCanvas: HTMLCanvasElement | null,
}
</script>
<script lang="ts">
import { loadImage } from "$lib/widgets/utils"
import { tick, createEventDispatcher } from "svelte";
import { ArrowClockwise, ArrowCounterclockwise, XSquare, Exclude, Circle, Grid3x3Gap, ArrowsFullscreen, FullscreenExit } from "svelte-bootstrap-icons";
export let fileURL: string | null = null;
export let fullscreen: boolean = false;
const dispatch = createEventDispatcher<{
change: MaskCanvasData;
release: MaskCanvasData;
}>();
let canvasCursor: string | undefined = undefined;
let container: HTMLDivElement | null;
let canvas: HTMLCanvasElement | null;
let maskCanvas: HTMLCanvasElement | null;
let renders: HTMLImageElement[] = [];
let context: CanvasRenderingContext2D | null;
let maskContext: CanvasRenderingContext2D | null;
let curLineGroup: LineGroup = [];
let redoCurLines: LineGroup = []
let original: HTMLImageElement | null;
let isImageLoaded: boolean = false;
let imageWidth: number = 512;
let imageHeight: number = 512;
let scale: number = 1.0;
let minScale: number = 1.0;
let brushSize: number = 100;
let maskBlur: number = 0;
let clipMask: boolean = false;
let hasMask: boolean = false;
let isDrawing: boolean = false;
let isPanning: boolean = false;
let isBrushShowing: boolean = false;
let transform = ""
$: transform = `translate(${imx}px, ${imy}px) scale(${scale})`
let imx: number = 0;
let imy: number = 0;
let x: number = 0;
let y: number = 0;
let panX: number = 0;
let panY: number = 0;
const BRUSH_COLOR = "#000"
enum MouseButton {
Left = 0,
Middle = 1,
Right = 2,
Back = 3,
Forward = 4
}
$: if (isPanning) {
canvasCursor = "grab";
}
else if (isImageLoaded && isBrushShowing) {
canvasCursor = "none";
}
else {
canvasCursor = undefined;
}
$: {
context = canvas ? canvas.getContext("2d") : null
}
function clearState() {
hasMask = false;
maskCanvas = null;
maskContext = null;
isImageLoaded = false;
original = null;
renders = []
curLineGroup = [];
redoCurLines = []
imageWidth = 512;
imageHeight = 512;
scale = 1.0;
minScale = 0.5;
}
let loadedFileURL: string | null = null
$: if (fileURL !== loadedFileURL) {
clearState();
if (fileURL) {
loadImage(fileURL).then(i => {
original = i;
isImageLoaded = true;
})
.catch(i => {
isImageLoaded = false;
})
}
else {
isImageLoaded = false;
original = null;
}
loadedFileURL = fileURL
}
$: {
// initSizeAndScale(isImageLoaded, original);
[imageWidth, imageHeight] = getCurrentWidthAndHeight(isImageLoaded, original)
scale = initScale(imageWidth, imageHeight)
initImagePos()
}
$: hasMask = curLineGroup.length > 0;
function initImagePos() {
if (!container)
return
const rect = container.getBoundingClientRect();
imx = rect.width / 2 - (imageWidth / 2) * scale
imy = rect.height / 2 - (imageHeight / 2) * scale
}
function initScale(width: number, height: number): number {
const s = getScale(width, height);
minScale = s / 2;
return s;
}
function initSizeAndScale(isImageLoaded: boolean, original: HTMLImageElement | null) {
[imageWidth, imageHeight] = getCurrentWidthAndHeight(isImageLoaded, original)
scale = getScale(imageWidth, imageHeight)
minScale = scale;
}
type LinePoint = {
x: number,
y: number
}
interface Line {
size?: number,
points: LinePoint[]
}
type LineGroup = Line[];
function drawOnCurrentRender(lineGroup: LineGroup) {
draw(lineGroup)
dispatch("change", {
hasMask,
maskCanvas
})
}
function draw(lineGroup: LineGroup) {
if (!context || !maskContext)
return
context.clearRect(0, 0, context.canvas.width, context.canvas.height)
maskContext.clearRect(0, 0, maskContext.canvas.width, maskContext.canvas.height)
const color = BRUSH_COLOR
const drawMask = (ctx: CanvasRenderingContext2D) => {
ctx.save();
ctx.filter = `blur(${maskBlur}px)`
drawLines(ctx, lineGroup, color)
ctx.restore();
}
drawMask(maskContext);
if (clipMask) {
context.save();
context.filter = `blur(${maskBlur}px)`
drawLines(context, lineGroup, color)
context.restore();
context.globalCompositeOperation = "source-in"
context.drawImage(original!!, 0, 0, imageWidth, imageHeight)
context.globalCompositeOperation = "source-over";
}
else {
drawMask(context);
}
}
function updateMaskImage() {
drawOnCurrentRender(curLineGroup);
}
function drawLines(ctx: CanvasRenderingContext2D, lines: LineGroup, color: string) {
ctx.strokeStyle = color
ctx.lineCap = 'round'
ctx.lineJoin = 'round'
lines.forEach(line => {
if (!line?.points.length || !line.size) {
return
}
ctx.lineWidth = line.size
ctx.beginPath()
ctx.moveTo(line.points[0].x, line.points[0].y)
line.points.forEach(point => ctx.lineTo(point.x, point.y))
ctx.stroke()
})
}
$: if (canvas && original) {
console.warn("INITCANVAS", imageWidth, imageHeight, original.src)
maskCanvas = document.createElement("canvas");
maskContext = maskCanvas.getContext("2d")!;
maskCanvas.width = imageWidth;
maskCanvas.height = imageHeight;
canvas.width = imageWidth;
canvas.height = imageHeight;
drawOnCurrentRender([])
}
function getCurrentWidthAndHeight(isImageLoaded: boolean, original: HTMLImageElement | null) {
if (isImageLoaded && original){
return [original.naturalWidth, original.naturalHeight]
}
return [512, 512]
}
function getScale(width: number, height: number): number {
const size = container?.getBoundingClientRect();
if (!size) {
return 1.0
}
const ratioWidth = size.width / width
const ratioHeight = (size.height) / height
let scale: number = 1.0
if (ratioWidth < 1 || ratioHeight < 1) {
scale = Math.min(ratioWidth, ratioHeight)
}
return scale
}
function undoStroke() {
if (curLineGroup.length === 0) {
return
}
const lastLine = curLineGroup.pop()!
const newRedoCurLines = [...redoCurLines, lastLine]
redoCurLines = newRedoCurLines
const newLineGroup = [...curLineGroup]
curLineGroup = newLineGroup
drawOnCurrentRender(newLineGroup)
}
function redoStroke() {
if (redoCurLines.length === 0) {
return
}
const line = redoCurLines.pop()!
redoCurLines = [...redoCurLines]
const newLineGroup = [...curLineGroup, line]
curLineGroup = newLineGroup
drawOnCurrentRender(newLineGroup)
}
export function clearStrokes() {
redoCurLines = []
const newLineGroup: LineGroup = []
curLineGroup = newLineGroup
drawOnCurrentRender(newLineGroup)
}
export function recenterImage() {
scale = initScale(imageWidth, imageHeight)
initImagePos();
}
async function toggleFullscreen() {
fullscreen = !fullscreen;
updateMaskImage();
await tick();
recenterImage();
}
function onCanvasMouseOver() {
isBrushShowing = true;
}
function onCanvasFocus() {
isBrushShowing = true;
}
function onCanvasMouseLeave() {
isBrushShowing = false;
}
function mouseXY(e: MouseEvent): LinePoint {
return { x: e.offsetX, y: e.offsetY }
}
function onCanvasMouseDown(e: MouseEvent) {
if (!original?.src)
return;
if (isPanning)
return;
if (canvas == null)
return;
switch (e.button) {
case MouseButton.Right:
return;
case MouseButton.Middle:
isPanning = true;
panX = e.offsetX * scale;
panY = e.offsetY * scale;
return;
}
isDrawing = true;
redoCurLines = []
let lineGroup: LineGroup = [...curLineGroup]
lineGroup.push({size: brushSize, points: [mouseXY(e)] })
curLineGroup = lineGroup
drawOnCurrentRender(curLineGroup);
}
function onCanvasMouseUp() {
}
function onCanvasMouseMove(e: MouseEvent) {
if (isPanning)
return;
if (!isDrawing)
return;
if (curLineGroup.length === 0)
return
curLineGroup[curLineGroup.length-1].points.push(mouseXY(e))
curLineGroup = curLineGroup; // react
drawOnCurrentRender(curLineGroup)
}
function onCanvasMouseWheel(e: WheelEvent) {
e.preventDefault();
if (!container || e.target != canvas)
return;
const bound = container.getBoundingClientRect()
// coodinates on the image that were zoomed
const x_ = e.clientX - bound.x
const y_ = e.clientY - bound.y
e.preventDefault();
var delta = e.deltaY * -0.001
delta = Math.max(-1,Math.min(1,delta)) // cap the delta to [-1,1] for cross browser consistency
const zx = (x_ - imx)/scale
const zy = (y_ - imy)/scale
scale += delta * scale
scale = Math.max(minScale,scale)
imx = -zx * scale + x_
imy = -zy * scale + y_
x = e.offsetX * scale;
y = e.offsetY * scale;
}
function onMouseMove(e: MouseEvent) {
if (e.target != canvas)
return;
x = e.offsetX * scale;
y = e.offsetY * scale;
if (isPanning) {
imx += x - panX;
imy += y - panY;
}
}
function onMouseUp(e: MouseEvent) {
if (e.button === MouseButton.Middle) {
isPanning = false
panX = 0
panY = 0
}
if (isPanning)
return;
if (!original?.src)
return;
if (!canvas)
return;
if (!isDrawing)
return;
isDrawing = false;
dispatch("release", { hasMask, maskCanvas })
}
function dispatchRelease() {
dispatch("release", { hasMask, maskCanvas })
}
</script>
<svelte:window on:mouseup={onMouseUp} />
<div class="me-container" class:fullscreen bind:this={container} on:mousemove={onMouseMove}>
{#if !isImageLoaded}
<div>
(empty)
</div>
{:else}
<div class="me-transform" style:transform={transform} style:--scale={scale}>
<div class="me-canvas-container">
<div class="me-original-image-container"
style:width="{imageWidth}px"
style:height="{imageHeight}px">
{#if original}
{@const showOriginal = !clipMask}
<img class="me-original-image"
src={original.src}
style:width={imageWidth}
style:height={imageHeight}
style:display={showOriginal ? "block" : "none"}
/>
{/if}
</div>
<canvas class="me-canvas"
bind:this={canvas}
style:cursor={canvasCursor}
on:mouseover={onCanvasMouseOver}
on:focus={onCanvasFocus}
on:wheel={onCanvasMouseWheel}
on:mouseleave={onCanvasMouseLeave}
on:mousedown|preventDefault={onCanvasMouseDown}
on:mouseup|preventDefault={onCanvasMouseUp}
on:mousemove={onCanvasMouseMove}
/>
</div>
</div>
{/if}
{#if isImageLoaded && isBrushShowing && !isPanning}
<div class="me-brush-cursor"
style:width="{brushSize * scale}px"
style:height="{brushSize * scale}px"
style:left="{x + imx}px"
style:top="{y + imy}px"
style:transform="translate(-50%, -50%)"
/>
{/if}
<div class="me-toolkit-bar">
<button disabled={curLineGroup.length === 0} on:click={undoStroke}>
<ArrowCounterclockwise />
</button>
<button disabled={redoCurLines.length === 0} on:click={redoStroke}>
<ArrowClockwise />
</button>
<button on:click={clearStrokes} disabled={curLineGroup.length === 0 && redoCurLines.length === 0}>
<XSquare/>
</button>
<label>
<Circle />
<input type="range" min="1" max="200" bind:value={brushSize} step="0.1"
on:change={updateMaskImage}
on:pointerup={dispatchRelease}/>
</label>
<label>
<Grid3x3Gap/>
<input type="range" min="1" max="100" bind:value={maskBlur} step="0.1"
on:change={updateMaskImage}
on:pointerup={dispatchRelease}/>
</label>
<div class="toggle-button" class:toggled={clipMask} on:click={() => {clipMask = !clipMask; updateMaskImage()}}>
<Exclude />
</div>
<div class="toggle-button" class:toggled={fullscreen} on:click={() => {toggleFullscreen()}}>
{#if fullscreen}
<FullscreenExit />
{:else}
<ArrowsFullscreen />
{/if}
</div>
</div>
</div>
<style lang="scss">
$bg-color: #a0a0a0;
.me-container {
width: 100%;
height: 100%;
position: relative;
overflow: hidden;
background-color: white;
background-image:
linear-gradient(45deg, #ccc 25%, transparent 25%),
linear-gradient(135deg, #ccc 25%, transparent 25%),
linear-gradient(45deg, transparent 75%, #ccc 75%),
linear-gradient(135deg, transparent 75%, #ccc 75%);
background-size:25px 25px; /* Must be a square */
background-position:0 0, 12.5px 0, 12.5px -12.5px, 0px 12.5px; /* Must be half of one side of the square */
&.fullscreen {
position: fixed;
top: 0;
left: 0;
width: 100vw;
height: 100vh;
margin: auto;
z-index: var(--layer-top);
}
}
.me-transform {
--scale: 1;
display: flex;
flex-wrap: wrap;
width: -moz-fit-content;
width: fit-content;
height: -moz-fit-content;
height: fit-content;
margin: 0;
padding: 0;
transform-origin: 0% 0%;
}
.me-original-image-container {
position: absolute;
top: 0;
left: 0;
grid-area: editor-content;
pointer-events: none;
user-select: none;
display: grid;
grid-template-areas: 'original-image-content';
border: calc((1 / var(--scale)) * 5px) dashed grey;
img.me-original-image {
grid-area: original-image-content;
}
}
.me-canvas {
position: absolute;
z-index: 1;
}
.me-brush-cursor {
position: absolute;
border-radius: 50%;
background-color: #000;
border: 1px solid var(--yellow-accent);
pointer-events: none;
}
.me-toolkit-bar {
position: absolute;
bottom: 0.5rem;
border-radius: 3rem;
padding: 0.4rem 24px;
display: flex;
margin: 0.5rem auto;
gap: 16px;
left: 0;
right: 0;
width: 80%;
height: 3rem;
align-items: center;
justify-content: space-evenly;
backdrop-filter: blur(12px);
background-color: white;
animation: slideUp 0.2s ease-out;
border: var(--editor-toolkit-panel-border);
box-shadow: 0 0 0 4px #0000001a, 0 3px 16px #00000014, 0 2px 6px 1px #00000017;
label {
display: flex;
flex-direction: row;
gap: 4px;
input {
width: 5rem;
}
}
button {
&:not(:disabled) {
cursor: pointer;
}
&:hover:not(:disabled) {
color: var(--secondary-600);
}
&:disabled {
opacity: 40%;
}
}
.toggle-button {
&:hover:not(:disabled) {
color: var(--secondary-600);
}
&:not(:disabled) {
cursor: pointer;
}
&.toggled {
color: var(--secondary-400);
}
}
}
</style>

59
src/lib/nodes/ComfyPickImageNode.ts
View File

@@ -1,8 +1,17 @@
import { LiteGraph, type ITextWidget, type SlotLayout, type INumberWidget } from "@litegraph-ts/core";
import ComfyGraphNode from "./ComfyGraphNode";
import ComfyGraphNode, { type ComfyGraphNodeProperties } from "./ComfyGraphNode";
import { comfyFileToAnnotatedFilepath, type ComfyBoxImageMetadata } from "$lib/utils";
export interface ComfyPickImageProperties extends ComfyGraphNodeProperties {
imageTagFilter: string
}
export default class ComfyPickImageNode extends ComfyGraphNode {
override properties: ComfyPickImageProperties = {
tags: [],
imageTagFilter: ""
}
static slotLayout: SlotLayout = {
inputs: [
{ name: "images", type: "COMFYBOX_IMAGES,COMFYBOX_IMAGE" },
@@ -13,57 +22,87 @@ export default class ComfyPickImageNode extends ComfyGraphNode {
{ name: "filename", type: "string" },
{ name: "width", type: "number" },
{ name: "height", type: "number" },
{ name: "children", type: "COMFYBOX_IMAGES" },
]
}
tagFilterWidget: ITextWidget;
filepathWidget: ITextWidget;
folderWidget: ITextWidget;
widthWidget: INumberWidget;
heightWidget: INumberWidget;
tagsWidget: ITextWidget;
childrenWidget: INumberWidget;
constructor(title?: string) {
super(title)
this.tagFilterWidget = this.addWidget("text", "Tag Filter", this.properties.imageTagFilter, "imageTagFilter")
this.filepathWidget = this.addWidget("text", "File", "")
this.filepathWidget.disabled = true;
this.folderWidget = this.addWidget("text", "Folder", "")
this.folderWidget.disabled = true;
this.widthWidget = this.addWidget("number", "Width", 0)
this.widthWidget.disabled = true;
this.heightWidget = this.addWidget("number", "Height", 0)
for (const widget of this.widgets)
widget.disabled = true;
this.heightWidget.disabled = true;
this.tagsWidget = this.addWidget("text", "Tags", "")
this.tagsWidget.disabled = true;
this.childrenWidget = this.addWidget("number", "# of Children", 0)
this.childrenWidget.disabled = true;
}
_value: ComfyBoxImageMetadata[] | null = null;
_image: ComfyBoxImageMetadata | null = null;
_path: string | null = null;
_index: number = 0;
_index: number | null = null;
private setValue(value: ComfyBoxImageMetadata[] | ComfyBoxImageMetadata | null, index: number) {
if (value != null && !Array.isArray(value)) {
value = [value]
index = 0;
}
const changed = this._value != value || this._index != index;
this._value = value as ComfyBoxImageMetadata[];
this._index = index;
let image: ComfyBoxImageMetadata | null = null;
if (value && this._index != null && value[this._index] != null) {
image = value[this._index];
}
const changed = this._value != value || this._index != index || this._image != image;
if (changed) {
if (value && value[this._index] != null) {
this._image = value[this._index]
if (image) {
this._image = image
this._image.children ||= []
this._image.tags ||= []
this._path = comfyFileToAnnotatedFilepath(this._image.comfyUIFile);
this.filepathWidget.value = this._image.comfyUIFile.filename
this.folderWidget.value = this._image.comfyUIFile.type
this.childrenWidget.value = this._image.children.length
this.tagsWidget.value = this._image.tags.join(", ")
}
else {
this._image = null;
this._path = null;
this.filepathWidget.value = "(None)"
this.folderWidget.value = ""
this.childrenWidget.value = 0
this.tagsWidget.value = ""
}
console.log("SET", value, this._image, this._path)
console.log("SET", value, this._image, this._path, this.properties.imageTagFilter)
}
}
override onExecute() {
const data = this.getInputData(0)
const index = this.getInputData(1) || 0
let index = this.getInputData(1);
if (this.properties.imageTagFilter && Array.isArray(data))
index = data.findIndex(i => i.tags?.includes(this.properties.imageTagFilter))
else
index = 0;
this.setValue(data, index);
if (this._image == null) {
@@ -71,6 +110,7 @@ export default class ComfyPickImageNode extends ComfyGraphNode {
this.setOutputData(1, null)
this.setOutputData(2, 0)
this.setOutputData(3, 0)
this.setOutputData(4, null)
this.widthWidget.value = 0
this.heightWidget.value = 0
@@ -80,6 +120,7 @@ export default class ComfyPickImageNode extends ComfyGraphNode {
this.setOutputData(1, this._path);
this.setOutputData(2, this._image.width);
this.setOutputData(3, this._image.height);
this.setOutputData(4, this._image.children);
// XXX: image size doesn't load until the <img> element is ready on
// the page so this can come after several frames' worth of

12
src/lib/nodes/widgets/ComfyImageUploadNode.ts
View File

@@ -8,12 +8,14 @@ import ComfyWidgetNode from "./ComfyWidgetNode";
import { get, writable, type Writable } from "svelte/store";
export interface ComfyImageUploadNodeProperties extends ComfyWidgetProperties {
maskCount: number
}
export default class ComfyImageUploadNode extends ComfyWidgetNode<ComfyBoxImageMetadata[]> {
properties: ComfyImageUploadNodeProperties = {
defaultValue: [],
tags: [],
maskCount: 0
}
static slotLayout: SlotLayout = {
@@ -39,13 +41,21 @@ export default class ComfyImageUploadNode extends ComfyWidgetNode<ComfyBoxImageM
super(name, [])
}
override onExecute() {
override onExecute(param: any, options: object) {
// TODO better way of getting image size?
const value = get(this.value)
if (value && value.length > 0) {
value[0].width = get(this.imgWidth)
value[0].height = get(this.imgHeight)
// NOTE: assumes masks will have the same image size as the parent image!
for (const child of value[0].children) {
child.width = get(this.imgWidth)
child.height = get(this.imgHeight)
}
}
super.onExecute(param, options);
}
override parseValue(value: any): ComfyBoxImageMetadata[] {

13
src/lib/stores/layoutStates.ts
View File

@@ -616,6 +616,19 @@ const ALL_ATTRIBUTES: AttributesSpecList = [
defaultValue: true
},
// ImageUpload
{
name: "maskCount",
type: "number",
location: "nodeProps",
editable: true,
validNodeTypes: ["ui/image_upload"],
defaultValue: 0,
min: 0,
max: 8,
step: 1
},
// Radio
{
name: "defaultValue",

78
src/lib/utils.ts
View File

@@ -75,6 +75,13 @@ export function download(filename: string, text: string, type: string = "text/pl
}, 0);
}
export function downloadCanvas(canvas: HTMLCanvasElement, filename: string, type: string = "image/png") {
var link = document.createElement('a');
link.download = filename;
link.href = canvas.toDataURL(type);
link.click();
}
export const MAX_LOCAL_STORAGE_MB = 5;
export function getLocalStorageUsedMB(): number {
@@ -434,6 +441,8 @@ export type ComfyBoxImageMetadata = {
width?: number,
/* Image height. */
height?: number,
/* Child images associated with this image, like masks. */
children: ComfyBoxImageMetadata[]
}
export function isComfyBoxImageMetadata(value: any): value is ComfyBoxImageMetadata {
@@ -458,6 +467,7 @@ export function filenameToComfyBoxMetadata(filename: string, type: ComfyUploadIm
},
name: "Filename",
tags: [],
children: []
}
}
@@ -467,6 +477,7 @@ export function comfyFileToComfyBoxMetadata(comfyUIFile: ComfyImageLocation): Co
comfyUIFile,
name: "File",
tags: [],
children: []
}
}
@@ -628,3 +639,70 @@ export function playSound(sound: string) {
const audio = new Audio(url);
audio.play();
}
export interface ComfyBatchUploadResult {
error?: string;
files: Array<ComfyImageLocation>;
}
export type ComfyBatchBlob = {
blob: Blob,
filename: string,
overwrite?: boolean
}
export async function batchUploadFilesToComfyUI(files: Array<File>): Promise<ComfyBatchUploadResult> {
const blobs = files.map(f => { return { blob: f, filename: f.name } })
return batchUploadBlobsToComfyUI(blobs)
}
export async function batchUploadBlobsToComfyUI(blobs: ComfyBatchBlob[]): Promise<ComfyBatchUploadResult> {
const url = configState.getBackendURL();
const requests = blobs.map(async (blob) => {
const formData = new FormData();
formData.append("image", blob.blob, blob.filename);
if (blob.overwrite) {
formData.append("overwrite", "true")
}
return fetch(new Request(url + "/upload/image", {
body: formData,
method: 'POST'
}))
.then(r => r.json())
.catch(error => error);
});
return Promise.all(requests)
.then((results) => {
const errors = []
const files = []
for (const r of results) {
if (r instanceof Error) {
errors.push(r.toString())
}
else {
// bare filename of image
const resp = r as ComfyUploadImageAPIResponse;
files.push({
filename: resp.name,
subfolder: "",
type: "input"
})
}
}
let error = null;
if (errors && errors.length > 0)
error = "Upload error(s):\n" + errors.join("\n");
return { error, files }
})
}
export function canvasToBlob(canvas: HTMLCanvasElement): Promise<Blob> {
return new Promise(function(resolve) {
canvas.toBlob(resolve);
});
}

331
src/lib/widgets/ImageUploadWidget.svelte
View File

@@ -3,46 +3,143 @@
import { Block } from "@gradio/atoms";
import { TextBox } from "@gradio/form";
import Row from "$lib/components/gradio/app/Row.svelte";
import { get, writable, type Writable } from "svelte/store";
import Modal from "$lib/components/Modal.svelte";
import { writable, type Writable } from "svelte/store";
import { Button } from "@gradio/button";
import { type Embed as Klecks } from "klecks";
import "klecks/style/style.scss";
import ImageUpload from "$lib/components/ImageUpload.svelte";
import { uploadImageToComfyUI, type ComfyBoxImageMetadata, comfyFileToComfyBoxMetadata, comfyBoxImageToComfyURL, comfyBoxImageToComfyFile, type ComfyUploadImageType, type ComfyImageLocation } from "$lib/utils";
import configState from "$lib/stores/configState";
import {
type ComfyBoxImageMetadata,
comfyFileToComfyBoxMetadata,
comfyBoxImageToComfyFile,
type ComfyImageLocation,
comfyBoxImageToComfyURL,
convertComfyOutputToComfyURL,
batchUploadBlobsToComfyUI,
canvasToBlob,
basename
} from "$lib/utils";
import notify from "$lib/notify";
import NumberInput from "$lib/components/NumberInput.svelte";
import type { ComfyImageEditorNode } from "$lib/nodes/widgets";
import { ImageViewer } from "$lib/ImageViewer";
import { generateBlankCanvas, generateImageCanvas } from "./utils";
import { ImageViewer } from "$lib/ImageViewer";
import MaskCanvas, { type MaskCanvasData } from "$lib/components/MaskCanvas.svelte";
import type { ComfyImageUploadNode } from "$lib/nodes/widgets";
import { tick } from "svelte";
export let widget: WidgetLayout | null = null;
export let isMobile: boolean = false;
let node: ComfyImageEditorNode | null = null;
let node: ComfyImageUploadNode | null = null;
let nodeValue: Writable<ComfyBoxImageMetadata[]> | null = null;
let attrsChanged: Writable<number> | null = null;
let imgWidth: Writable<number> = writable(0);
let imgHeight: Writable<number> = writable(0);
let maskCanvasComp: MaskCanvas | null = null;
let editMask: boolean = false;
$: widget && setNodeValue(widget);
let canMask = false;
$: canMask = (node?.properties?.maskCount || 0) > 0;
$: if (!canMask) clearMask();
function setNodeValue(widget: WidgetLayout) {
if (widget) {
node = widget.node as ComfyImageEditorNode
node = widget.node as ComfyImageUploadNode
nodeValue = node.value;
attrsChanged = widget.attrsChanged;
imgWidth = node.imgWidth
imgHeight = node.imgHeight
status = $nodeValue && $nodeValue.length > 0 ? "uploaded" : "empty"
}
};
let hasImage = false;
$: hasImage = $nodeValue && $nodeValue.length > 0;
$: if (!hasImage) {
editMask = false;
}
const MASK_FILENAME: string = "ComfyBoxMask.png"
async function onMaskReleased(e: CustomEvent<MaskCanvasData>) {
const data = e.detail;
if (data.maskCanvas && data.hasMask) {
await saveMask(data.maskCanvas)
}
}
async function saveMask(maskCanvas: HTMLCanvasElement) {
if (!canMask) {
notify("Mask editing is disabled for this widget.", { type: "warning" })
return;
}
if (!maskCanvas) {
notify("No mask canvas!", { type: "warning" })
return
}
if (!$nodeValue || $nodeValue.length === 0) {
notify("No image uploaded to apply mask to.", { type: "warning" })
return
}
const hadNoMask = $nodeValue[0].children.findIndex(i => i.tags?.includes("mask")) === -1;
const existFilename = $nodeValue[0].comfyUIFile.filename
const filename = existFilename ? `${basename(existFilename)}_mask.png` : MASK_FILENAME
console.warn("[ImageUpload] UPLOAD MASK", filename)
await canvasToBlob(maskCanvas)
.then(blob => batchUploadBlobsToComfyUI([{
blob,
filename,
overwrite: true
}]))
.then(result => {
const meta = result.files.map(f => {
const m = comfyFileToComfyBoxMetadata(f)
m.tags = ["mask"]
m.width = maskCanvas.width;
m.height = maskCanvas.height;
return m;
});
if ($nodeValue.length > 0) {
// TODO support multiple images?
$nodeValue[0].children = meta;
if (hadNoMask) {
notify("Uploaded mask successfully!", { type: "success" })
}
}
else {
throw new Error("No image was uploaded yet.")
}
})
.catch(error => {
notify(`Failed to upload mask to ComfyUI: ${error}`, { type: "error", timeout: 10000 })
})
}
function clearMask() {
for (const image of $nodeValue) {
// TODO other child image types preserved here?
image.children = [];
}
if (maskCanvasComp) {
maskCanvasComp.clearStrokes();
}
}
async function toggleEditMask() {
editMask = !editMask;
await tick();
if (maskCanvasComp) {
maskCanvasComp.recenterImage();
}
}
let editorRoot: HTMLDivElement | null = null;
let showModal = false;
let kl: Klecks | null = null;
function disposeEditor() {
console.warn("[ImageEditorWidget] CLOSING", widget, $nodeValue)
@@ -53,100 +150,9 @@
}
}
kl = null;
showModal = false;
}
const FILENAME: string = "ComfyUITemp.png";
const SUBFOLDER: string = "ComfyBox_Editor";
const DIRECTORY: ComfyUploadImageType = "input";
async function submitKlecksToComfyUI(onSuccess: () => void, onError: () => void) {
const blob = kl.getPNG();
status = "uploading"
await uploadImageToComfyUI(blob, FILENAME, DIRECTORY, SUBFOLDER)
.then((entry: ComfyImageLocation) => {
const meta: ComfyBoxImageMetadata = comfyFileToComfyBoxMetadata(entry);
$nodeValue = [meta] // TODO more than one image
status = "uploaded"
notify("Saved image to ComfyUI!", { type: "success" })
onSuccess();
})
.catch(err => {
notify(`Failed to upload image from editor: ${err}`, { type: "error", timeout: 10000 })
status = "error"
uploadError = err;
$nodeValue = []
onError();
})
}
let closeDialog = null;
async function saveAndClose() {
console.log(closeDialog, kl)
if (!closeDialog || !kl)
return;
submitKlecksToComfyUI(() => {}, () => {});
closeDialog()
}
let blankImageWidth = 512;
let blankImageHeight = 512;
let klecks: typeof import("klecks") | null = null;
async function openImageEditor() {
if (!editorRoot)
return;
showModal = true;
const url = configState.getBackendURL();
klecks ||= await import("klecks");
kl = new klecks.Embed({
embedUrl: url,
onSubmit: submitKlecksToComfyUI,
targetEl: editorRoot,
warnOnPageClose: false
});
console.warn("[ImageEditorWidget] OPENING", widget, $nodeValue)
let canvas = null;
let width = blankImageWidth;
let height = blankImageHeight;
if ($nodeValue && $nodeValue.length > 0) {
const comfyImage = $nodeValue[0];
const comfyURL = comfyBoxImageToComfyURL(comfyImage);
[canvas, width, height] = await generateImageCanvas(comfyURL);
}
else {
canvas = generateBlankCanvas(width, height);
}
kl.openProject({
width: width,
height: height,
layers: [{
name: 'Image',
opacity: 1,
mixModeStr: 'source-over',
image: canvas
}]
});
setTimeout(function () {
kl?.klApp?.out("yo");
}, 1000);
}
function openLightbox() {
if (!$nodeValue || $nodeValue.length === 0)
return;
@@ -189,9 +195,6 @@
notify(`Failed to upload image to ComfyUI: ${uploadError}`, { type: "error", timeout: 10000 })
}
function onChange(e: CustomEvent<ComfyImageLocation[]>) {
}
let _value: ComfyImageLocation[] = []
$: if ($nodeValue)
_value = $nodeValue.map(comfyBoxImageToComfyFile)
@@ -199,6 +202,10 @@
_value = []
$: canEdit = status === "empty" || status === "uploaded";
function onChange(e: CustomEvent<ComfyImageLocation[]>) {
}
</script>
<div class="wrapper comfy-image-editor">
@@ -219,64 +226,50 @@
/>
{:else}
<div class="comfy-image-editor-panel">
<ImageUpload value={_value}
bind:imgWidth={$imgWidth}
bind:imgHeight={$imgHeight}
fileCount={"single"}
elem_classes={[]}
style={""}
label={widget.attrs.title}
on:uploading={onUploading}
on:uploaded={onUploaded}
on:upload_error={onUploadError}
on:clear={onClear}
on:change={onChange}
on:image_clicked={openLightbox}
/>
<Modal bind:showModal closeOnClick={false} on:close={disposeEditor} bind:closeDialog>
<div>
<div id="klecks-loading-screen">
<span id="klecks-loading-screen-text"></span>
</div>
<div class="image-editor-root" bind:this={editorRoot} />
{#if _value && canMask}
{@const comfyURL = convertComfyOutputToComfyURL(_value[0])}
<div class="mask-canvas-wrapper" style:display={editMask ? "block" : "none"}>
<MaskCanvas bind:this={maskCanvasComp} fileURL={comfyURL} on:release={onMaskReleased} />
</div>
<div slot="buttons">
<Button variant="primary" on:click={saveAndClose}>
Save and Close
</Button>
<Button variant="secondary" on:click={closeDialog}>
Discard Edits
</Button>
</div>
</Modal>
{/if}
<div style:display={(canMask && editMask) ? "none" : "block"}>
<ImageUpload value={_value}
bind:imgWidth={$imgWidth}
bind:imgHeight={$imgHeight}
fileCount={"single"}
elem_classes={[]}
style={""}
label={widget.attrs.title}
on:uploading={onUploading}
on:uploaded={onUploaded}
on:upload_error={onUploadError}
on:clear={onClear}
on:change={onChange}
on:image_clicked={openLightbox}
/>
</div>
<Block>
{#if !$nodeValue || $nodeValue.length === 0}
{#if hasImage}
{@const maskCount = $nodeValue[0] ? $nodeValue[0].children.filter(f => f.tags?.includes("mask")).length : 0}
<Row>
<Row>
<Button variant="secondary" disabled={!canEdit} on:click={openImageEditor}>
Create Image
</Button>
{#if canMask}
<div>
<TextBox show_label={false} disabled={true} value="Status: {status}"/>
{#if editMask}
<Button variant="secondary" on:click={() => { clearMask(); notify("Mask cleared."); }}>
Clear Mask
</Button>
{/if}
<Button disabled={!_value} on:click={toggleEditMask}>
{#if editMask}
Show Image
{:else}
Edit Mask
{/if}
</Button>
</div>
{#if uploadError}
<div>
Upload error: {uploadError}
</div>
{/if}
</Row>
<Row>
<NumberInput label={"Width"} min={64} max={2048} step={64} bind:value={blankImageWidth} />
<NumberInput label={"Height"} min={64} max={2048} step={64} bind:value={blankImageHeight} />
</Row>
</Row>
{:else}
<Row>
<Button variant="secondary" disabled={!canEdit} on:click={openImageEditor}>
Edit Image
</Button>
{/if}
<div>
<TextBox label={""} show_label={false} disabled={true} lines={1} max_lines={1} value="Status: {status}"/>
<TextBox label={""} show_label={false} disabled={true} lines={1} max_lines={1} value="Images: {$nodeValue.length}, masks: {maskCount}"/>
</div>
{#if uploadError}
<div>
@@ -291,25 +284,13 @@
</div>
<style lang="scss">
.image-editor-root {
width: 75vw;
height: 75vh;
overflow: hidden;
color: black;
:global(> .g-root) {
height: calc(100% - 59px);
}
}
.comfy-image-editor {
:global(> dialog) {
overflow: hidden;
}
}
:global(.kl-popup) {
z-index: 999999999999;
.mask-canvas-wrapper {
height: calc(var(--size-96) * 1.5);
}
</style>