Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Sign in / Register
Toggle navigation
S
Stable Diffusion Webui
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Locked Files
Issues
0
Issues
0
List
Boards
Labels
Service Desk
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Security & Compliance
Security & Compliance
Dependency List
License Compliance
Packages
Packages
List
Container Registry
Analytics
Analytics
CI / CD
Code Review
Insights
Issues
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
novelai-storage
Stable Diffusion Webui
Commits
c7a86f7f
Commit
c7a86f7f
authored
Oct 15, 2022
by
AUTOMATIC
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
add option to use batch size for training
parent
acedbe67
Changes
4
Hide whitespace changes
Inline
Side-by-side
Showing
4 changed files
with
54 additions
and
30 deletions
+54
-30
modules/hypernetworks/hypernetwork.py
modules/hypernetworks/hypernetwork.py
+24
-9
modules/textual_inversion/dataset.py
modules/textual_inversion/dataset.py
+19
-12
modules/textual_inversion/textual_inversion.py
modules/textual_inversion/textual_inversion.py
+8
-9
modules/ui.py
modules/ui.py
+3
-0
No files found.
modules/hypernetworks/hypernetwork.py
View file @
c7a86f7f
...
...
@@ -182,7 +182,21 @@ def attention_CrossAttention_forward(self, x, context=None, mask=None):
return
self
.
to_out
(
out
)
def
train_hypernetwork
(
hypernetwork_name
,
learn_rate
,
data_root
,
log_directory
,
steps
,
create_image_every
,
save_hypernetwork_every
,
template_file
,
preview_from_txt2img
,
preview_prompt
,
preview_negative_prompt
,
preview_steps
,
preview_sampler_index
,
preview_cfg_scale
,
preview_seed
,
preview_width
,
preview_height
):
def
stack_conds
(
conds
):
if
len
(
conds
)
==
1
:
return
torch
.
stack
(
conds
)
# same as in reconstruct_multicond_batch
token_count
=
max
([
x
.
shape
[
0
]
for
x
in
conds
])
for
i
in
range
(
len
(
conds
)):
if
conds
[
i
]
.
shape
[
0
]
!=
token_count
:
last_vector
=
conds
[
i
][
-
1
:]
last_vector_repeated
=
last_vector
.
repeat
([
token_count
-
conds
[
i
]
.
shape
[
0
],
1
])
conds
[
i
]
=
torch
.
vstack
([
conds
[
i
],
last_vector_repeated
])
return
torch
.
stack
(
conds
)
def
train_hypernetwork
(
hypernetwork_name
,
learn_rate
,
batch_size
,
data_root
,
log_directory
,
steps
,
create_image_every
,
save_hypernetwork_every
,
template_file
,
preview_from_txt2img
,
preview_prompt
,
preview_negative_prompt
,
preview_steps
,
preview_sampler_index
,
preview_cfg_scale
,
preview_seed
,
preview_width
,
preview_height
):
assert
hypernetwork_name
,
'hypernetwork not selected'
path
=
shared
.
hypernetworks
.
get
(
hypernetwork_name
,
None
)
...
...
@@ -211,7 +225,7 @@ def train_hypernetwork(hypernetwork_name, learn_rate, data_root, log_directory,
shared
.
state
.
textinfo
=
f
"Preparing dataset from {html.escape(data_root)}..."
with
torch
.
autocast
(
"cuda"
):
ds
=
modules
.
textual_inversion
.
dataset
.
PersonalizedBase
(
data_root
=
data_root
,
width
=
512
,
height
=
512
,
repeats
=
shared
.
opts
.
training_image_repeats_per_epoch
,
placeholder_token
=
hypernetwork_name
,
model
=
shared
.
sd_model
,
device
=
devices
.
device
,
template_file
=
template_file
,
include_cond
=
True
)
ds
=
modules
.
textual_inversion
.
dataset
.
PersonalizedBase
(
data_root
=
data_root
,
width
=
512
,
height
=
512
,
repeats
=
shared
.
opts
.
training_image_repeats_per_epoch
,
placeholder_token
=
hypernetwork_name
,
model
=
shared
.
sd_model
,
device
=
devices
.
device
,
template_file
=
template_file
,
include_cond
=
True
,
batch_size
=
batch_size
)
if
unload
:
shared
.
sd_model
.
cond_stage_model
.
to
(
devices
.
cpu
)
...
...
@@ -235,7 +249,7 @@ def train_hypernetwork(hypernetwork_name, learn_rate, data_root, log_directory,
optimizer
=
torch
.
optim
.
AdamW
(
weights
,
lr
=
scheduler
.
learn_rate
)
pbar
=
tqdm
.
tqdm
(
enumerate
(
ds
),
total
=
steps
-
ititial_step
)
for
i
,
entr
y
in
pbar
:
for
i
,
entr
ies
in
pbar
:
hypernetwork
.
step
=
i
+
ititial_step
scheduler
.
apply
(
optimizer
,
hypernetwork
.
step
)
...
...
@@ -246,11 +260,12 @@ def train_hypernetwork(hypernetwork_name, learn_rate, data_root, log_directory,
break
with
torch
.
autocast
(
"cuda"
):
cond
=
entry
.
cond
.
to
(
devices
.
device
)
x
=
entry
.
latent
.
to
(
devices
.
device
)
loss
=
shared
.
sd_model
(
x
.
unsqueeze
(
0
),
cond
)[
0
]
c
=
stack_conds
([
entry
.
cond
for
entry
in
entries
])
.
to
(
devices
.
device
)
# c = torch.vstack([entry.cond for entry in entries]).to(devices.device)
x
=
torch
.
stack
([
entry
.
latent
for
entry
in
entries
])
.
to
(
devices
.
device
)
loss
=
shared
.
sd_model
(
x
,
c
)[
0
]
del
x
del
c
ond
del
c
losses
[
hypernetwork
.
step
%
losses
.
shape
[
0
]]
=
loss
.
item
()
...
...
@@ -292,7 +307,7 @@ def train_hypernetwork(hypernetwork_name, learn_rate, data_root, log_directory,
p
.
width
=
preview_width
p
.
height
=
preview_height
else
:
p
.
prompt
=
entr
y
.
cond_text
p
.
prompt
=
entr
ies
[
0
]
.
cond_text
p
.
steps
=
20
preview_text
=
p
.
prompt
...
...
@@ -315,7 +330,7 @@ def train_hypernetwork(hypernetwork_name, learn_rate, data_root, log_directory,
<p>
Loss: {losses.mean():.7f}<br/>
Step: {hypernetwork.step}<br/>
Last prompt: {html.escape(entr
y
.cond_text)}<br/>
Last prompt: {html.escape(entr
ies[0]
.cond_text)}<br/>
Last saved embedding: {html.escape(last_saved_file)}<br/>
Last saved image: {html.escape(last_saved_image)}<br/>
</p>
...
...
modules/textual_inversion/dataset.py
View file @
c7a86f7f
...
...
@@ -24,11 +24,12 @@ class DatasetEntry:
class
PersonalizedBase
(
Dataset
):
def
__init__
(
self
,
data_root
,
width
,
height
,
repeats
,
flip_p
=
0.5
,
placeholder_token
=
"*"
,
model
=
None
,
device
=
None
,
template_file
=
None
,
include_cond
=
False
):
re_word
=
re
.
compile
(
shared
.
opts
.
dataset_filename_word_regex
)
if
len
(
shared
.
opts
.
dataset_filename_word_regex
)
>
0
else
None
def
__init__
(
self
,
data_root
,
width
,
height
,
repeats
,
flip_p
=
0.5
,
placeholder_token
=
"*"
,
model
=
None
,
device
=
None
,
template_file
=
None
,
include_cond
=
False
,
batch_size
=
1
):
re_word
=
re
.
compile
(
shared
.
opts
.
dataset_filename_word_regex
)
if
len
(
shared
.
opts
.
dataset_filename_word_regex
)
>
0
else
None
self
.
placeholder_token
=
placeholder_token
self
.
batch_size
=
batch_size
self
.
width
=
width
self
.
height
=
height
self
.
flip
=
transforms
.
RandomHorizontalFlip
(
p
=
flip_p
)
...
...
@@ -78,13 +79,13 @@ class PersonalizedBase(Dataset):
if
include_cond
:
entry
.
cond_text
=
self
.
create_text
(
filename_text
)
entry
.
cond
=
cond_model
([
entry
.
cond_text
])
.
to
(
devices
.
cpu
)
entry
.
cond
=
cond_model
([
entry
.
cond_text
])
.
to
(
devices
.
cpu
)
.
squeeze
(
0
)
self
.
dataset
.
append
(
entry
)
self
.
length
=
len
(
self
.
dataset
)
*
repeats
self
.
length
=
len
(
self
.
dataset
)
*
repeats
//
batch_size
self
.
initial_indexes
=
np
.
arange
(
self
.
length
)
%
len
(
self
.
dataset
)
self
.
initial_indexes
=
np
.
arange
(
len
(
self
.
dataset
)
)
self
.
indexes
=
None
self
.
shuffle
()
...
...
@@ -101,13 +102,19 @@ class PersonalizedBase(Dataset):
return
self
.
length
def
__getitem__
(
self
,
i
):
if
i
%
len
(
self
.
dataset
)
==
0
:
self
.
shuffle
()
res
=
[]
index
=
self
.
indexes
[
i
%
len
(
self
.
indexes
)]
entry
=
self
.
dataset
[
index
]
for
j
in
range
(
self
.
batch_size
):
position
=
i
*
self
.
batch_size
+
j
if
position
%
len
(
self
.
indexes
)
==
0
:
self
.
shuffle
()
if
entry
.
cond
is
None
:
entry
.
cond_text
=
self
.
create_text
(
entry
.
filename_text
)
index
=
self
.
indexes
[
position
%
len
(
self
.
indexes
)]
entry
=
self
.
dataset
[
index
]
return
entry
if
entry
.
cond
is
None
:
entry
.
cond_text
=
self
.
create_text
(
entry
.
filename_text
)
res
.
append
(
entry
)
return
res
modules/textual_inversion/textual_inversion.py
View file @
c7a86f7f
...
...
@@ -199,7 +199,7 @@ def write_loss(log_directory, filename, step, epoch_len, values):
})
def
train_embedding
(
embedding_name
,
learn_rate
,
data_root
,
log_directory
,
training_width
,
training_height
,
steps
,
create_image_every
,
save_embedding_every
,
template_file
,
save_image_with_stored_embedding
,
preview_from_txt2img
,
preview_prompt
,
preview_negative_prompt
,
preview_steps
,
preview_sampler_index
,
preview_cfg_scale
,
preview_seed
,
preview_width
,
preview_height
):
def
train_embedding
(
embedding_name
,
learn_rate
,
batch_size
,
data_root
,
log_directory
,
training_width
,
training_height
,
steps
,
create_image_every
,
save_embedding_every
,
template_file
,
save_image_with_stored_embedding
,
preview_from_txt2img
,
preview_prompt
,
preview_negative_prompt
,
preview_steps
,
preview_sampler_index
,
preview_cfg_scale
,
preview_seed
,
preview_width
,
preview_height
):
assert
embedding_name
,
'embedding not selected'
shared
.
state
.
textinfo
=
"Initializing textual inversion training..."
...
...
@@ -231,7 +231,7 @@ def train_embedding(embedding_name, learn_rate, data_root, log_directory, traini
shared
.
state
.
textinfo
=
f
"Preparing dataset from {html.escape(data_root)}..."
with
torch
.
autocast
(
"cuda"
):
ds
=
modules
.
textual_inversion
.
dataset
.
PersonalizedBase
(
data_root
=
data_root
,
width
=
training_width
,
height
=
training_height
,
repeats
=
shared
.
opts
.
training_image_repeats_per_epoch
,
placeholder_token
=
embedding_name
,
model
=
shared
.
sd_model
,
device
=
devices
.
device
,
template_file
=
template_file
)
ds
=
modules
.
textual_inversion
.
dataset
.
PersonalizedBase
(
data_root
=
data_root
,
width
=
training_width
,
height
=
training_height
,
repeats
=
shared
.
opts
.
training_image_repeats_per_epoch
,
placeholder_token
=
embedding_name
,
model
=
shared
.
sd_model
,
device
=
devices
.
device
,
template_file
=
template_file
,
batch_size
=
batch_size
)
hijack
=
sd_hijack
.
model_hijack
...
...
@@ -251,7 +251,7 @@ def train_embedding(embedding_name, learn_rate, data_root, log_directory, traini
optimizer
=
torch
.
optim
.
AdamW
([
embedding
.
vec
],
lr
=
scheduler
.
learn_rate
)
pbar
=
tqdm
.
tqdm
(
enumerate
(
ds
),
total
=
steps
-
ititial_step
)
for
i
,
entr
y
in
pbar
:
for
i
,
entr
ies
in
pbar
:
embedding
.
step
=
i
+
ititial_step
scheduler
.
apply
(
optimizer
,
embedding
.
step
)
...
...
@@ -262,10 +262,9 @@ def train_embedding(embedding_name, learn_rate, data_root, log_directory, traini
break
with
torch
.
autocast
(
"cuda"
):
c
=
cond_model
([
entry
.
cond_text
])
x
=
entry
.
latent
.
to
(
devices
.
device
)
loss
=
shared
.
sd_model
(
x
.
unsqueeze
(
0
),
c
)[
0
]
c
=
cond_model
([
entry
.
cond_text
for
entry
in
entries
])
x
=
torch
.
stack
([
entry
.
latent
for
entry
in
entries
])
.
to
(
devices
.
device
)
loss
=
shared
.
sd_model
(
x
,
c
)[
0
]
del
x
losses
[
embedding
.
step
%
losses
.
shape
[
0
]]
=
loss
.
item
()
...
...
@@ -307,7 +306,7 @@ def train_embedding(embedding_name, learn_rate, data_root, log_directory, traini
p
.
width
=
preview_width
p
.
height
=
preview_height
else
:
p
.
prompt
=
entr
y
.
cond_text
p
.
prompt
=
entr
ies
[
0
]
.
cond_text
p
.
steps
=
20
p
.
width
=
training_width
p
.
height
=
training_height
...
...
@@ -348,7 +347,7 @@ def train_embedding(embedding_name, learn_rate, data_root, log_directory, traini
<p>
Loss: {losses.mean():.7f}<br/>
Step: {embedding.step}<br/>
Last prompt: {html.escape(entr
y
.cond_text)}<br/>
Last prompt: {html.escape(entr
ies[0]
.cond_text)}<br/>
Last saved embedding: {html.escape(last_saved_file)}<br/>
Last saved image: {html.escape(last_saved_image)}<br/>
</p>
...
...
modules/ui.py
View file @
c7a86f7f
...
...
@@ -1166,6 +1166,7 @@ def create_ui(wrap_gradio_gpu_call):
train_embedding_name
=
gr
.
Dropdown
(
label
=
'Embedding'
,
choices
=
sorted
(
sd_hijack
.
model_hijack
.
embedding_db
.
word_embeddings
.
keys
()))
train_hypernetwork_name
=
gr
.
Dropdown
(
label
=
'Hypernetwork'
,
choices
=
[
x
for
x
in
shared
.
hypernetworks
.
keys
()])
learn_rate
=
gr
.
Textbox
(
label
=
'Learning rate'
,
placeholder
=
"Learning rate"
,
value
=
"0.005"
)
batch_size
=
gr
.
Number
(
label
=
'Batch size'
,
value
=
1
,
precision
=
0
)
dataset_directory
=
gr
.
Textbox
(
label
=
'Dataset directory'
,
placeholder
=
"Path to directory with input images"
)
log_directory
=
gr
.
Textbox
(
label
=
'Log directory'
,
placeholder
=
"Path to directory where to write outputs"
,
value
=
"textual_inversion"
)
template_file
=
gr
.
Textbox
(
label
=
'Prompt template file'
,
value
=
os
.
path
.
join
(
script_path
,
"textual_inversion_templates"
,
"style_filewords.txt"
))
...
...
@@ -1244,6 +1245,7 @@ def create_ui(wrap_gradio_gpu_call):
inputs
=
[
train_embedding_name
,
learn_rate
,
batch_size
,
dataset_directory
,
log_directory
,
training_width
,
...
...
@@ -1268,6 +1270,7 @@ def create_ui(wrap_gradio_gpu_call):
inputs
=
[
train_hypernetwork_name
,
learn_rate
,
batch_size
,
dataset_directory
,
log_directory
,
steps
,
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment