From 27d0a4bd30018980c3f09f65b867555ce59daef5 Mon Sep 17 00:00:00 2001
From: space-nuko <24979496+space-nuko@users.noreply.github.com>
Date: Mon, 5 Jun 2023 21:15:49 -0500
Subject: [PATCH] Thumbnail option for gallery/queue images

---
 src/lib/components/ComfyQueue.svelte          |  21 +--
 .../components/ComfyQueueGridDisplay.svelte   |   7 +-
 .../components/ComfyQueueListDisplay.svelte   |   8 +-
 src/lib/components/PromptDisplay.svelte       |  11 +-
 src/lib/stores/configDefs.ts                  |  31 ++++
 src/lib/stores/uiQueueState.ts                |  11 +-
 src/lib/utils.ts                              | 139 ++++++++++++++----
 src/mobile/routes/gallery.svelte              |  16 +-
 src/mobile/routes/queue.svelte                |   4 +-
 9 files changed, 174 insertions(+), 74 deletions(-)

diff --git a/src/lib/components/ComfyQueue.svelte b/src/lib/components/ComfyQueue.svelte
index 31719e2..a9c50a9 100644
--- a/src/lib/components/ComfyQueue.svelte
+++ b/src/lib/components/ComfyQueue.svelte
@@ -1,18 +1,3 @@
-<script lang="ts" context="module">
- export type QueueUIEntryStatus = QueueEntryStatus | "pending" | "running";
-
- export type QueueUIEntry = {
-     entry: QueueEntry,
-     message: string,
-     submessage: string,
-     date?: string,
-     status: QueueUIEntryStatus,
-     images?: string[], // URLs
-     details?: string, // shown in a tooltip on hover
-     error?: WorkflowError
- }
-</script>
-
 <script lang="ts">
  import queueState, { type CompletedQueueEntry, type QueueEntry, type QueueEntryStatus } from "$lib/stores/queueState";
  import ProgressBar from "./ProgressBar.svelte";
@@ -20,7 +5,7 @@
  import Spinner from "./Spinner.svelte";
  import PromptDisplay from "./PromptDisplay.svelte";
  import { List, ListUl, Grid } from "svelte-bootstrap-icons";
- import { convertComfyOutputToComfyURL, convertFilenameToComfyURL, getNodeInfo, truncateString } from "$lib/utils"
+ import { getNodeInfo, type ComfyImageLocation } from "$lib/utils"
  import type { Writable } from "svelte/store";
  import type { QueueItemType } from "$lib/api";
  import { Button } from "@gradio/button";
@@ -31,7 +16,7 @@
  import ComfyQueueListDisplay from "./ComfyQueueListDisplay.svelte";
  import ComfyQueueGridDisplay from "./ComfyQueueGridDisplay.svelte";
 	import { WORKFLOWS_VIEW } from "./ComfyBoxWorkflowsView.svelte";
-	import uiQueueState from "$lib/stores/uiQueueState";
+	import uiQueueState, { type QueueUIEntry } from "$lib/stores/uiQueueState";
 
  export let app: ComfyApp;
 
@@ -125,7 +110,7 @@
  let showModal = false;
  let expandAll = false;
  let selectedPrompt = null;
- let selectedImages = [];
+ let selectedImages: ComfyImageLocation[] = [];
  function showPrompt(entry: QueueUIEntry) {
      if (entry.error != null) {
          showModal = false;
diff --git a/src/lib/components/ComfyQueueGridDisplay.svelte b/src/lib/components/ComfyQueueGridDisplay.svelte
index 8ca6509..ce8e5c3 100644
--- a/src/lib/components/ComfyQueueGridDisplay.svelte
+++ b/src/lib/components/ComfyQueueGridDisplay.svelte
@@ -1,6 +1,6 @@
 <script lang="ts">
  import type { QueueItemType } from "$lib/api";
- import { showLightbox } from "$lib/utils";
+ import { convertComfyOutputToComfyURL, showLightbox } from "$lib/utils";
  import type { QueueUIEntry } from "./ComfyQueue.svelte";
  import queueState from "$lib/stores/queueState";
 
@@ -19,7 +19,7 @@
      allEntries = []
      for (const entry of entries) {
          for (const image of entry.images) {
-             allEntries.push([entry, image]);
+             allEntries.push([entry, convertComfyOutputToComfyURL(image, true)]);
          }
      }
      allImages = allEntries.map(p => p[1]);
@@ -56,6 +56,7 @@
                     <img class="grid-entry-image"
                          on:click={(e) => handleClick(e, entry, i)}
                     src={image}
+                    loading="lazy"
                     alt="thumbnail" />
                 </div>
             {/each}
@@ -130,6 +131,8 @@
  .grid-entry-image {
      aspect-ratio: 1 / 1;
      object-fit: cover;
+     width: 100%;
+     max-width: unset;
 
      &:hover {
          cursor: pointer;
diff --git a/src/lib/components/ComfyQueueListDisplay.svelte b/src/lib/components/ComfyQueueListDisplay.svelte
index 751a9f8..d8dd4ab 100644
--- a/src/lib/components/ComfyQueueListDisplay.svelte
+++ b/src/lib/components/ComfyQueueListDisplay.svelte
@@ -1,8 +1,8 @@
 <script lang="ts">
  import type { QueueItemType } from "$lib/api";
- import { showLightbox, truncateString } from "$lib/utils";
- import type { QueueUIEntry } from "./ComfyQueue.svelte";
+ import { convertComfyOutputToComfyURL, showLightbox, truncateString } from "$lib/utils";
  import queueState from "$lib/stores/queueState";
+ import type { QueueUIEntry } from "$lib/stores/uiQueueState";
 
  export let entries: QueueUIEntry[] = [];
  export let showPrompt: (entry: QueueUIEntry) => void;
@@ -39,11 +39,13 @@
                     <div class="list-entry-images"
                          style="--cols: {Math.ceil(Math.sqrt(Math.min(entry.images.length, 4)))}" >
                         {#each entry.images.slice(0, 4) as image, i}
+                            {@const imageURL = convertComfyOutputToComfyURL(image, true)}
                             <div>
                                 <!-- svelte-ignore a11y-click-events-have-key-events -->
                                 <img class="list-entry-image"
                                      on:click={(e) => showLightbox(entry.images, i, e)}
-                                src={image}
+                                src={imageURL}
+                                loading="lazy"
                                 alt="thumbnail" />
                             </div>
                         {/each}
diff --git a/src/lib/components/PromptDisplay.svelte b/src/lib/components/PromptDisplay.svelte
index b96e2c7..ee9de16 100644
--- a/src/lib/components/PromptDisplay.svelte
+++ b/src/lib/components/PromptDisplay.svelte
@@ -8,7 +8,7 @@
  import Gallery from "$lib/components/gradio/gallery/Gallery.svelte";
  import { ImageViewer } from "$lib/ImageViewer";
  import type { Styles } from "@gradio/utils";
- import { comfyFileToComfyBoxMetadata, comfyURLToComfyFile, countNewLines } from "$lib/utils";
+ import { comfyFileToComfyBoxMetadata, comfyURLToComfyFile, countNewLines, type ComfyImageLocation, convertComfyOutputToComfyURL } from "$lib/utils";
  import ReceiveOutputTargets from "./modal/ReceiveOutputTargets.svelte";
  import workflowState, { type ComfyBoxWorkflow, type WorkflowReceiveOutputTargets } from "$lib/stores/workflowState";
  import type { ComfyReceiveOutputNode } from "$lib/nodes/actions";
@@ -17,7 +17,7 @@
  const splitLength = 50;
 
  export let prompt: SerializedPromptInputsAll;
- export let images: string[] = []; // list of image URLs to ComfyUI's /view? endpoint
+ export let images: ComfyImageLocation[] = [];
  export let isMobile: boolean = false;
  export let expandAll: boolean = false;
  export let closeModal: () => void;
@@ -36,10 +36,7 @@
  let litegraphType = "(none)"
 
  $: if (images.length > 0) {
-     // since the image links come from gradio, have to parse the URL for the
-     // ComfyImageLocation params
-     comfyBoxImages = images.map(comfyURLToComfyFile)
-                            .map(comfyFileToComfyBoxMetadata);
+     comfyBoxImages = images.map(comfyFileToComfyBoxMetadata);
  }
  else {
      comfyBoxImages = []
@@ -199,7 +196,7 @@
         <div class="image-container">
             <Block>
                 <Gallery
-                    value={images}
+                    value={images.map(convertComfyOutputToComfyURL)}
                     label=""
                     show_label={false}
                     style={galleryStyle}
diff --git a/src/lib/stores/configDefs.ts b/src/lib/stores/configDefs.ts
index 4cd07b2..01ceb92 100644
--- a/src/lib/stores/configDefs.ts
+++ b/src/lib/stores/configDefs.ts
@@ -119,6 +119,36 @@ const defNotifications: ConfigDefEnum<"notifications", NotificationState> = {
     }
 };
 
+export enum OutputThumbnailsMode {
+    Auto,
+    AlwaysThumbnail,
+    AlwaysFullSize
+}
+
+const defOutputThumbnails: ConfigDefEnum<"outputThumbnails", OutputThumbnailsMode> = {
+    name: "outputThumbnails",
+    type: "enum",
+    defaultValue: OutputThumbnailsMode.Auto,
+    category: "ui",
+    description: "If enabled, send back smaller sized output image thumbnails for gallery/queue/history. Enable if you have slow network or are using Colab.",
+    options: {
+        values: [
+            {
+                value: OutputThumbnailsMode.Auto,
+                label: "Autodetect"
+            },
+            {
+                value: OutputThumbnailsMode.AlwaysThumbnail,
+                label: "Always use thumbnails"
+            },
+            {
+                value: OutputThumbnailsMode.AlwaysFullSize,
+                label: "Always use full size"
+            },
+        ]
+    }
+};
+
 const defAlwaysStripUserState: ConfigDefBoolean<"alwaysStripUserState"> = {
     name: "alwaysStripUserState",
     type: "boolean",
@@ -207,6 +237,7 @@ export const CONFIG_DEFS = [
     defComfyUIHostname,
     defComfyUIPort,
     defNotifications,
+    defOutputThumbnails,
     defAlwaysStripUserState,
     defPromptForWorkflowName,
     defConfirmWhenUnloadingUnsavedChanges,
diff --git a/src/lib/stores/uiQueueState.ts b/src/lib/stores/uiQueueState.ts
index a361c22..d16af8a 100644
--- a/src/lib/stores/uiQueueState.ts
+++ b/src/lib/stores/uiQueueState.ts
@@ -1,7 +1,8 @@
 import type { PromptID, QueueItemType } from '$lib/api';
+import type { ComfyImageLocation } from "$lib/utils";
 import { get, writable } from 'svelte/store';
 import type { Readable, Writable } from 'svelte/store';
-import queueState, { type CompletedQueueEntry, type QueueEntry } from './queueState';
+import queueState, { QueueEntryStatus, type CompletedQueueEntry, type QueueEntry } from './queueState';
 import type { WorkflowError } from './workflowState';
 import { convertComfyOutputToComfyURL } from '$lib/utils';
 
@@ -13,7 +14,7 @@ export type QueueUIEntry = {
     submessage: string,
     date?: string,
     status: QueueUIEntryStatus,
-    images?: string[], // URLs
+    images?: ComfyImageLocation[], // URLs
     details?: string, // shown in a tooltip on hover
     error?: WorkflowError
 }
@@ -94,13 +95,12 @@ function convertPendingEntry(entry: QueueEntry, status: QueueUIEntryStatus): Que
 
     const thumbnails = entry.extraData?.thumbnails
     if (thumbnails) {
-        result.images = thumbnails.map(convertComfyOutputToComfyURL);
+        result.images = [...thumbnails]
     }
 
     const outputs = Object.values(entry.outputs)
         .filter(o => o.images)
         .flatMap(o => o.images)
-        .map(convertComfyOutputToComfyURL);
     if (outputs) {
         result.images = result.images.concat(outputs)
     }
@@ -114,7 +114,6 @@ function convertCompletedEntry(entry: CompletedQueueEntry): QueueUIEntry {
     const images = Object.values(entry.entry.outputs)
         .filter(o => o.images)
         .flatMap(o => o.images)
-        .map(convertComfyOutputToComfyURL);
     result.images = images
 
     if (entry.message)
@@ -132,6 +131,8 @@ function updateFromQueue(queuePending: QueueEntry[], queueRunning: QueueEntry[])
         // newest entries appear at the top
         s.queuedEntries = queuePending.map((e) => convertPendingEntry(e, "pending")).reverse();
         s.runningEntries = queueRunning.map((e) => convertPendingEntry(e, "running")).reverse();
+        s.queuedEntries.sort((a, b) => a.entry.number - b.entry.number)
+        s.runningEntries.sort((a, b) => a.entry.number - b.entry.number)
         s.queueUIEntries = s.queuedEntries.concat(s.runningEntries);
         console.warn("[ComfyQueue] BUILDQUEUE", s.queuedEntries.length, s.runningEntries.length)
         return s;
diff --git a/src/lib/utils.ts b/src/lib/utils.ts
index 68bdff3..7b1df82 100644
--- a/src/lib/utils.ts
+++ b/src/lib/utils.ts
@@ -9,6 +9,7 @@ import workflowState, { type WorkflowReceiveOutputTargets } from "./stores/workf
 import { ImageViewer } from "./ImageViewer";
 import configState from "$lib/stores/configState";
 import SendOutputModal, { type SendOutputModalResult } from "$lib/components/modal/SendOutputModal.svelte";
+import { OutputThumbnailsMode } from "./stores/configDefs";
 
 export function clamp(n: number, min: number, max: number): number {
     if (max <= min)
@@ -300,31 +301,80 @@ export function convertComfyOutputToGradio(output: SerializedPromptOutput): Grad
 
 export function convertComfyOutputEntryToGradio(r: ComfyImageLocation): GradioFileData {
     const url = configState.getBackendURL();
-    const params = new URLSearchParams(r)
     const fileData: GradioFileData = {
         name: r.filename,
         orig_name: r.filename,
         is_file: false,
-        data: url + "/view?" + params
+        data: convertComfyOutputToComfyURL(r)
     }
     return fileData
 }
 
-export function convertComfyOutputToComfyURL(output: string | ComfyImageLocation): string {
+function convertComfyPreviewTypeToString(preview: ComfyImagePreviewType): string {
+    const arr = []
+    switch (preview.format) {
+        case ComfyImagePreviewFormat.JPEG:
+            arr.push("jpeg")
+            break;
+        case ComfyImagePreviewFormat.WebP:
+        default:
+            arr.push("webp")
+            break;
+    }
+
+    arr.push(String(preview.quality))
+
+    return arr.join(";")
+}
+
+export function convertComfyOutputToComfyURL(output: string | ComfyImageLocation, thumbnail: boolean = false): string {
     if (typeof output === "string")
         return output;
 
-    const params = new URLSearchParams(output)
+    const paramsObj = {
+        filename: output.filename,
+        subfolder: output.subfolder,
+        type: output.type
+    }
+
+    if (thumbnail) {
+        let doThumbnail: boolean;
+
+        switch (get(configState).outputThumbnails) {
+            case OutputThumbnailsMode.AlwaysFullSize:
+                doThumbnail = false;
+                break;
+            case OutputThumbnailsMode.AlwaysThumbnail:
+                doThumbnail = true;
+                break;
+            case OutputThumbnailsMode.Auto:
+            default:
+                // TODO detect colab, etc.
+                if (isMobileBrowser(navigator.userAgent)) {
+                    doThumbnail = true;
+                }
+                else {
+                    doThumbnail = false;
+                }
+                break;
+        }
+
+        if (doThumbnail) {
+            output.preview = {
+                format: ComfyImagePreviewFormat.WebP,
+                quality: 80
+            }
+        }
+    }
+
+    if (output.preview != null)
+        paramsObj["preview"] = convertComfyPreviewTypeToString(output.preview)
+
+    const params = new URLSearchParams(paramsObj)
     const url = configState.getBackendURL();
     return url + "/view?" + params
 }
 
-export function convertGradioFileDataToComfyURL(image: GradioFileData, type: ComfyUploadImageType = "input"): string {
-    const baseUrl = configState.getBackendURL();
-    const params = new URLSearchParams({ filename: image.name, subfolder: "", type })
-    return `${baseUrl}/view?${params}`
-}
-
 export function convertGradioFileDataToComfyOutput(fileData: GradioFileData, type: ComfyUploadImageType = "input"): ComfyImageLocation {
     if (!fileData.is_file)
         throw "Can't convert blob data to comfy output!"
@@ -336,18 +386,6 @@ export function convertGradioFileDataToComfyOutput(fileData: GradioFileData, typ
     }
 }
 
-export function convertFilenameToComfyURL(filename: string,
-    subfolder: string = "",
-    type: "input" | "output" | "temp" = "output"): string {
-    const params = new URLSearchParams({
-        filename,
-        subfolder,
-        type
-    })
-    const url = configState.getBackendURL();
-    return url + "/view?" + params
-}
-
 export function jsonToJsObject(json: string): string {
     // Try to parse, to see if it's real JSON
     JSON.parse(json);
@@ -413,6 +451,16 @@ export interface SerializedPromptOutput {
     [key: string]: any
 }
 
+export enum ComfyImagePreviewFormat {
+    WebP = "webp",
+    JPEG = "jpeg",
+}
+
+export type ComfyImagePreviewType = {
+    format: ComfyImagePreviewFormat,
+    quality: number
+}
+
 /** Raw output entry as received from ComfyUI's backend */
 export type ComfyImageLocation = {
     /* Filename with extension in the subfolder. */
@@ -420,7 +468,19 @@ export type ComfyImageLocation = {
     /* Subfolder in the containing folder. */
     subfolder: string,
     /* Base ComfyUI folder where the image is located. */
-    type: ComfyUploadImageType
+    type: ComfyUploadImageType,
+    /*
+     * Preview information
+     *
+     * "format;quality"
+     *
+     * ex)
+     * webp;50 -> webp, quality 50
+     * webp;50 -> webp, quality 50
+     * jpeg;80 -> rgb, jpeg, quality 80
+     *
+     */
+    preview?: ComfyImagePreviewType
 }
 
 /*
@@ -544,27 +604,54 @@ export function comfyBoxImageToComfyURL(image: ComfyBoxImageMetadata): string {
     return convertComfyOutputToComfyURL(image.comfyUIFile)
 }
 
+function parseComfyUIPreviewType(previewStr: string): ComfyImagePreviewType {
+    let split = previewStr.split(";")
+    let format = ComfyImagePreviewFormat.WebP;
+    if (split[0] === "webp")
+        format = ComfyImagePreviewFormat.WebP;
+    else if (split[0] === "jpeg")
+        format = ComfyImagePreviewFormat.JPEG;
+
+    let quality = parseInt(split[0])
+    if (isNaN(quality))
+        quality = 80
+
+    return { format, quality }
+}
+
 export function comfyURLToComfyFile(urlString: string): ComfyImageLocation | null {
     const url = new URL(urlString);
     const params = new URLSearchParams(url.search);
     const filename = params.get("filename")
     const type = params.get("type") as ComfyUploadImageType;
     const subfolder = params.get("subfolder") || ""
+    const previewStr = params.get("preview") || null;
+    let preview = null
+
+    if (previewStr != null) {
+        preview = parseComfyUIPreviewType(preview);
+    }
 
     // If at least filename and type exist then we're good
     if (filename != null && type != null) {
-        return { filename, type, subfolder }
+        return { filename, type, subfolder, preview }
     }
 
     return null;
 }
 
-export function showLightbox(images: string[], index: number, e: Event) {
+export function showLightbox(images: ComfyImageLocation[] | string[], index: number, e: Event) {
     e.preventDefault()
     if (!images)
         return
 
-    ImageViewer.instance.showModal(images, index);
+    let images_: string[]
+    if (typeof images[0] === "object")
+        images_ = (images as ComfyImageLocation[]).map(v => convertComfyOutputToComfyURL(v))
+    else
+        images_ = (images as string[])
+
+    ImageViewer.instance.showModal(images_, index);
 
     e.stopPropagation()
 }
diff --git a/src/mobile/routes/gallery.svelte b/src/mobile/routes/gallery.svelte
index 9efe72a..5c40a5a 100644
--- a/src/mobile/routes/gallery.svelte
+++ b/src/mobile/routes/gallery.svelte
@@ -1,17 +1,11 @@
 <script lang="ts">
  import { Page, Navbar, Block, Tabs, Tab, NavLeft, NavTitle, NavRight, Link, f7 } from "framework7-svelte"
- import WidgetContainer from "$lib/components/WidgetContainer.svelte";
  import type ComfyApp from "$lib/components/ComfyApp";
- import { writable, type Writable } from "svelte/store";
- import type { IDragItem, WritableLayoutStateStore } from "$lib/stores/layoutStates";
- import workflowState, { type ComfyBoxWorkflow, type WorkflowInstID } from "$lib/stores/workflowState";
  import interfaceState from "$lib/stores/interfaceState";
- import { onMount } from "svelte";
- import GenToolbar from '../GenToolbar.svelte'
- import { partition, showLightbox } from "$lib/utils";
-	import uiQueueState, { type QueueUIEntry } from "$lib/stores/uiQueueState";
-	import { showMobileLightbox } from "$lib/components/utils";
-	import notify from "$lib/notify";
+ import { convertComfyOutputToComfyURL, partition, showLightbox } from "$lib/utils";
+ import uiQueueState, { type QueueUIEntry } from "$lib/stores/uiQueueState";
+ import { showMobileLightbox } from "$lib/components/utils";
+ import notify from "$lib/notify";
 
  export let app: ComfyApp
 
@@ -33,7 +27,7 @@
      const _allEntries = []
      for (const entry of entries) {
          for (const image of entry.images) {
-             _allEntries.push([entry, image]);
+             _allEntries.push([entry, convertComfyOutputToComfyURL(image, true)]);
          }
      }
      allEntries = partition(_allEntries, gridCols);
diff --git a/src/mobile/routes/queue.svelte b/src/mobile/routes/queue.svelte
index a3f6ecd..1127436 100644
--- a/src/mobile/routes/queue.svelte
+++ b/src/mobile/routes/queue.svelte
@@ -8,7 +8,7 @@
  import interfaceState from "$lib/stores/interfaceState";
  import { onMount } from "svelte";
  import GenToolbar from '../GenToolbar.svelte'
- import { partition, showLightbox, truncateString } from "$lib/utils";
+ import { convertComfyOutputToComfyURL, partition, showLightbox, truncateString } from "$lib/utils";
 	import uiQueueState, { type QueueUIEntry } from "$lib/stores/uiQueueState";
 	import { showMobileLightbox } from "$lib/components/utils";
 	import queueState from "$lib/stores/queueState";
@@ -68,7 +68,7 @@
 
  function getCardImage(entry: QueueUIEntry): string {
      if (entry.images.length > 0)
-         return entry.images[0]
+         return convertComfyOutputToComfyURL(entry.images[0])
      return "https://cdn.framework7.io/placeholder/nature-1000x600-3.jpg"
  }
 </script>