Added ui sopport for multi control samples and datasets. Added qwen i…

…mage edit 5209 to the ui

config/examples/train_lora_qwen_image_edit_2509_32gb.yaml

@@ -0,0 +1,105 @@
+---
+job: extension
+config:
+  # this name will be the folder and filename name
+  name: "my_first_qwen_image_edit_2509_lora_v1"
+  process:
+    - type: 'diffusion_trainer'
+      # root folder to save training sessions/samples/weights
+      training_folder: "output"
+      # uncomment to see performance stats in the terminal every N steps
+#      performance_log_every: 1000
+      device: cuda:0
+      network:
+        type: "lora"
+        linear: 16
+        linear_alpha: 16
+      save:
+        dtype: float16 # precision to save
+        save_every: 250 # save every this many steps
+        max_step_saves_to_keep: 4 # how many intermittent saves to keep
+      datasets:
+        # datasets are a folder of images. captions need to be txt files with the same name as the image
+        # for instance image2.jpg and image2.txt. Only jpg, jpeg, and png are supported currently
+        # images will automatically be resized and bucketed into the resolution specified
+        # on windows, escape back slashes with another backslash so
+        # "C:\\path\\to\\images\\folder"
+        - folder_path: "/path/to/images/folder"
+          # can do up to 3 control image folders, file names must match target file names, but aspect/size can be different
+          control_path:
+            - "/path/to/control/images/folder1"
+            - "/path/to/control/images/folder2"
+            - "/path/to/control/images/folder3"
+          caption_ext: "txt"
+          # default_caption: "a person" # if caching text embeddings, if you don't have captions, this will get cached
+          caption_dropout_rate: 0.05  # will drop out the caption 5% of time
+          resolution: [ 512, 768, 1024 ]  # qwen image enjoys multiple resolutions
+          # a trigger word that can be cached with the text embeddings 
+          # trigger_word: "optional trigger word"
+      train:
+        batch_size: 1
+        # caching text embeddings is required for 32GB
+        cache_text_embeddings: true
+        # unload_text_encoder: true
+
+        steps: 3000  # total number of steps to train 500 - 4000 is a good range
+        gradient_accumulation: 1
+        timestep_type: "weighted"
+        train_unet: true
+        train_text_encoder: false  # probably won't work with qwen image
+        gradient_checkpointing: true  # need the on unless you have a ton of vram
+        noise_scheduler: "flowmatch" # for training only
+        optimizer: "adamw8bit"
+        lr: 1e-4
+        # uncomment this to skip the pre training sample
+        # skip_first_sample: true
+        # uncomment to completely disable sampling
+        # disable_sampling: true
+        dtype: bf16
+      model:
+        # huggingface model name or path
+        name_or_path: "Qwen/Qwen-Image-Edit-2509"
+        arch: "qwen_image_edit_plus"
+        quantize: true
+        # to use the ARA use the | pipe to point to hf path, or a local path if you have one.
+        # 3bit is required for 32GB
+        qtype: "uint3|ostris/accuracy_recovery_adapters/qwen_image_edit_2509_torchao_uint3.safetensors"
+        quantize_te: true
+        qtype_te: "qfloat8"
+        low_vram: true
+      sample:
+        sampler: "flowmatch" # must match train.noise_scheduler
+        sample_every: 250 # sample every this many steps
+        width: 1024
+        height: 1024
+        # you can provide up to 3 control images here
+        samples:
+          - prompt: "Do whatever with Image1 and Image2"
+            ctrl_img_1: "/path/to/image1.png"
+            ctrl_img_2: "/path/to/image2.png"
+            # ctrl_img_3: "/path/to/image3.png"
+          - prompt: "Do whatever with Image1 and Image2"
+            ctrl_img_1: "/path/to/image1.png"
+            ctrl_img_2: "/path/to/image2.png"
+            # ctrl_img_3: "/path/to/image3.png"
+          - prompt: "Do whatever with Image1 and Image2"
+            ctrl_img_1: "/path/to/image1.png"
+            ctrl_img_2: "/path/to/image2.png"
+            # ctrl_img_3: "/path/to/image3.png"
+          - prompt: "Do whatever with Image1 and Image2"
+            ctrl_img_1: "/path/to/image1.png"
+            ctrl_img_2: "/path/to/image2.png"
+            # ctrl_img_3: "/path/to/image3.png"
+          - prompt: "Do whatever with Image1 and Image2"
+            ctrl_img_1: "/path/to/image1.png"
+            ctrl_img_2: "/path/to/image2.png"
+            # ctrl_img_3: "/path/to/image3.png"
+        neg: ""
+        seed: 42
+        walk_seed: true
+        guidance_scale: 3
+        sample_steps: 25
+# you can add any additional meta info here. [name] is replaced with config name at top
+meta:
+  name: "[name]"
+  version: '1.0'

toolkit/config_modules.py

@@ -831,6 +831,21 @@ def __init__(self, **kwargs):
         if self.control_path == '':
             self.control_path = None
 
+        # handle multi control inputs from the ui. It is just easier to handle it here for a cleaner ui experience
+        control_path_1 = kwargs.get('control_path_1', None)
+        control_path_2 = kwargs.get('control_path_2', None)
+        control_path_3 = kwargs.get('control_path_3', None)
+
+        if any([control_path_1, control_path_2, control_path_3]):
+            control_paths = []
+            if control_path_1:
+                control_paths.append(control_path_1)
+            if control_path_2:
+                control_paths.append(control_path_2)
+            if control_path_3:
+                control_paths.append(control_path_3)
+            self.control_path = control_paths
+
         # color for transparent reigon of control images with transparency
         self.control_transparent_color: List[int] = kwargs.get('control_transparent_color', [0, 0, 0])
         # inpaint images should be webp/png images with alpha channel. The alpha 0 (invisible) section will

ui/src/app/jobs/new/SimpleJob.tsx

@@ -15,7 +15,9 @@ import { TextInput, SelectInput, Checkbox, FormGroup, NumberInput } from '@/comp
 import Card from '@/components/Card';
 import { X } from 'lucide-react';
 import AddSingleImageModal, { openAddImageModal } from '@/components/AddSingleImageModal';
+import SampleControlImage from '@/components/SampleControlImage';
 import { FlipHorizontal2, FlipVertical2 } from 'lucide-react';
+import { handleModelArchChange } from './utils';
 
 type Props = {
   jobConfig: JobConfig;
@@ -185,58 +187,7 @@ export default function SimpleJob({
               label="Model Architecture"
               value={jobConfig.config.process[0].model.arch}
               onChange={value => {
-                const currentArch = modelArchs.find(a => a.name === jobConfig.config.process[0].model.arch);
-                if (!currentArch || currentArch.name === value) {
-                  return;
-                }
-                // update the defaults when a model is selected
-                const newArch = modelArchs.find(model => model.name === value);
-
-                // update vram setting
-                if (!newArch?.additionalSections?.includes('model.low_vram')) {
-                  setJobConfig(false, 'config.process[0].model.low_vram');
-                }
-
-                // revert defaults from previous model
-                for (const key in currentArch.defaults) {
-                  setJobConfig(currentArch.defaults[key][1], key);
-                }
-
-                if (newArch?.defaults) {
-                  for (const key in newArch.defaults) {
-                    setJobConfig(newArch.defaults[key][0], key);
-                  }
-                }
-                // set new model
-                setJobConfig(value, 'config.process[0].model.arch');
-
-                // update datasets
-                const hasControlPath = newArch?.additionalSections?.includes('datasets.control_path') || false;
-                const hasNumFrames = newArch?.additionalSections?.includes('datasets.num_frames') || false;
-                const controls = newArch?.controls ?? [];
-                const datasets = jobConfig.config.process[0].datasets.map(dataset => {
-                  const newDataset = objectCopy(dataset);
-                  newDataset.controls = controls;
-                  if (!hasControlPath) {
-                    newDataset.control_path = null; // reset control path if not applicable
-                  }
-                  if (!hasNumFrames) {
-                    newDataset.num_frames = 1; // reset num_frames if not applicable
-                  }
-                  return newDataset;
-                });
-                setJobConfig(datasets, 'config.process[0].datasets');
-
-                // update samples
-                const hasSampleCtrlImg = newArch?.additionalSections?.includes('sample.ctrl_img') || false;
-                const samples = jobConfig.config.process[0].sample.samples.map(sample => {
-                  const newSample = objectCopy(sample);
-                  if (!hasSampleCtrlImg) {
-                    delete newSample.ctrl_img; // remove ctrl_img if not applicable
-                  }
-                  return newSample;
-                });
-                setJobConfig(samples, 'config.process[0].sample.samples');
+                handleModelArchChange(jobConfig.config.process[0].model.arch, value, jobConfig, setJobConfig);
               }}
               options={groupedModelOptions}
             />
@@ -557,17 +508,19 @@ export default function SimpleJob({
                 )}
 
                 <FormGroup label="Text Encoder Optimizations" className="pt-2">
-                  <Checkbox
-                    label="Unload TE"
-                    checked={jobConfig.config.process[0].train.unload_text_encoder || false}
-                    docKey={'train.unload_text_encoder'}
-                    onChange={value => {
-                      setJobConfig(value, 'config.process[0].train.unload_text_encoder');
-                      if (value) {
-                        setJobConfig(false, 'config.process[0].train.cache_text_embeddings');
-                      }
-                    }}
-                  />
+                  {!disableSections.includes('train.unload_text_encoder') && (
+                    <Checkbox
+                      label="Unload TE"
+                      checked={jobConfig.config.process[0].train.unload_text_encoder || false}
+                      docKey={'train.unload_text_encoder'}
+                      onChange={value => {
+                        setJobConfig(value, 'config.process[0].train.unload_text_encoder');
+                        if (value) {
+                          setJobConfig(false, 'config.process[0].train.cache_text_embeddings');
+                        }
+                      }}
+                    />
+                  )}
                   <Checkbox
                     label="Cache Text Embeddings"
                     checked={jobConfig.config.process[0].train.cache_text_embeddings || false}
@@ -642,7 +595,7 @@ export default function SimpleJob({
                   <div className="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-4 gap-6">
                     <div>
                       <SelectInput
-                        label="Dataset"
+                        label="Target Dataset"
                         value={dataset.folder_path}
                         onChange={value => setJobConfig(value, `config.process[0].datasets[${i}].folder_path`)}
                         options={datasetOptions}
@@ -659,6 +612,49 @@ export default function SimpleJob({
                           options={[{ value: '', label: <>&nbsp;</> }, ...datasetOptions]}
                         />
                       )}
+                      {modelArch?.additionalSections?.includes('datasets.multi_control_paths') && (
+                        <>
+                          <SelectInput
+                            label="Control Dataset 1"
+                            docKey="datasets.multi_control_paths"
+                            value={dataset.control_path_1 ?? ''}
+                            className="pt-2"
+                            onChange={value =>
+                              setJobConfig(
+                                value == '' ? null : value,
+                                `config.process[0].datasets[${i}].control_path_1`,
+                              )
+                            }
+                            options={[{ value: '', label: <>&nbsp;</> }, ...datasetOptions]}
+                          />
+                          <SelectInput
+                            label="Control Dataset 2"
+                            docKey="datasets.multi_control_paths"
+                            value={dataset.control_path_2 ?? ''}
+                            className="pt-2"
+                            onChange={value =>
+                              setJobConfig(
+                                value == '' ? null : value,
+                                `config.process[0].datasets[${i}].control_path_2`,
+                              )
+                            }
+                            options={[{ value: '', label: <>&nbsp;</> }, ...datasetOptions]}
+                          />
+                          <SelectInput
+                            label="Control Dataset 3"
+                            docKey="datasets.multi_control_paths"
+                            value={dataset.control_path_3 ?? ''}
+                            className="pt-2"
+                            onChange={value =>
+                              setJobConfig(
+                                value == '' ? null : value,
+                                `config.process[0].datasets[${i}].control_path_3`,
+                              )
+                            }
+                            options={[{ value: '', label: <>&nbsp;</> }, ...datasetOptions]}
+                          />
+                        </>
+                      )}
                       <NumberInput
                         label="LoRA Weight"
                         value={dataset.network_weight}
@@ -1062,30 +1058,43 @@ export default function SimpleJob({
                           />
                         </div>
                       </div>
-
+                      {modelArch?.additionalSections?.includes('datasets.multi_control_paths') && (
+                        <FormGroup label="Control Images" className="pt-2 ml-4">
+                          <div className="grid grid-cols-1 md:grid-cols-3 gap-2 mt-2 mt-2">
+                            {['ctrl_img_1', 'ctrl_img_2', 'ctrl_img_3'].map((ctrlKey, ctrl_idx) => (
+                              <SampleControlImage
+                                key={ctrlKey}
+                                instruction={`Add Control Image ${ctrl_idx + 1}`}
+                                className=""
+                                src={sample[ctrlKey as keyof typeof sample] as string}
+                                onNewImageSelected={imagePath => {
+                                  if (!imagePath) {
+                                    let newSamples = objectCopy(jobConfig.config.process[0].sample.samples);
+                                    delete newSamples[i][ctrlKey as keyof typeof sample];
+                                    setJobConfig(newSamples, 'config.process[0].sample.samples');
+                                  } else {
+                                    setJobConfig(imagePath, `config.process[0].sample.samples[${i}].${ctrlKey}`);
+                                  }
+                                }}
+                              />
+                            ))}
+                          </div>
+                        </FormGroup>
+                      )}
                       {modelArch?.additionalSections?.includes('sample.ctrl_img') && (
-                        <div
-                          className="h-14 w-14 mt-2 ml-4 border border-gray-500 flex items-center justify-center rounded cursor-pointer hover:bg-gray-700 transition-colors"
-                          style={{
-                            backgroundImage: sample.ctrl_img
-                              ? `url(${`/api/img/${encodeURIComponent(sample.ctrl_img)}`})`
-                              : 'none',
-                            backgroundSize: 'cover',
-                            backgroundPosition: 'center',
-                            marginBottom: '-1rem',
-                          }}
-                          onClick={() => {
-                            openAddImageModal(imagePath => {
-                              console.log('Selected image path:', imagePath);
-                              if (!imagePath) return;
+                        <SampleControlImage
+                          className="mt-6 ml-4"
+                          src={sample.ctrl_img}
+                          onNewImageSelected={imagePath => {
+                            if (!imagePath) {
+                              let newSamples = objectCopy(jobConfig.config.process[0].sample.samples);
+                              delete newSamples[i].ctrl_img;
+                              setJobConfig(newSamples, 'config.process[0].sample.samples');
+                            } else {
                               setJobConfig(imagePath, `config.process[0].sample.samples[${i}].ctrl_img`);
-                            });
+                            }
                           }}
-                        >
-                          {!sample.ctrl_img && (
-                            <div className="text-gray-400 text-xs text-center font-bold">Add Control Image</div>
-                          )}
-                        </div>
+                        />
                       )}
                     </div>
                     <div className="pb-4"></div>

ui/src/app/jobs/new/jobConfig.ts

@@ -2,7 +2,6 @@ import { JobConfig, DatasetConfig, SliderConfig } from '@/types';
 
 export const defaultDatasetConfig: DatasetConfig = {
   folder_path: '/path/to/images/folder',
-  control_path: null,
   mask_path: null,
   mask_min_value: 0.1,
   default_caption: '',