Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Sign in / Register
Toggle navigation
S
Stable Diffusion Webui
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Locked Files
Issues
0
Issues
0
List
Boards
Labels
Service Desk
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Security & Compliance
Security & Compliance
Dependency List
License Compliance
Packages
Packages
List
Container Registry
Analytics
Analytics
CI / CD
Code Review
Insights
Issues
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
novelai-storage
Stable Diffusion Webui
Commits
6ef0ff39
Commit
6ef0ff39
authored
Dec 14, 2023
by
AUTOMATIC1111
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Merge pull request #14300 from AUTOMATIC1111/oft_fixes
Fix wrong implementation in network_oft
parent
120a84bd
Changes
1
Show whitespace changes
Inline
Side-by-side
Showing
1 changed file
with
11 additions
and
26 deletions
+11
-26
extensions-builtin/Lora/network_oft.py
extensions-builtin/Lora/network_oft.py
+11
-26
No files found.
extensions-builtin/Lora/network_oft.py
View file @
6ef0ff39
...
...
@@ -21,6 +21,8 @@ class NetworkModuleOFT(network.NetworkModule):
self
.
lin_module
=
None
self
.
org_module
:
list
[
torch
.
Module
]
=
[
self
.
sd_module
]
self
.
scale
=
1.0
# kohya-ss
if
"oft_blocks"
in
weights
.
w
.
keys
():
self
.
is_kohya
=
True
...
...
@@ -53,12 +55,18 @@ class NetworkModuleOFT(network.NetworkModule):
self
.
constraint
=
None
self
.
block_size
,
self
.
num_blocks
=
factorization
(
self
.
out_dim
,
self
.
dim
)
def
calc_updown
_kb
(
self
,
orig_weight
,
multiplier
):
def
calc_updown
(
self
,
orig_weight
):
oft_blocks
=
self
.
oft_blocks
.
to
(
orig_weight
.
device
,
dtype
=
orig_weight
.
dtype
)
oft_blocks
=
oft_blocks
-
oft_blocks
.
transpose
(
1
,
2
)
# ensure skew-symmetric orthogonal matrix
eye
=
torch
.
eye
(
self
.
block_size
,
device
=
self
.
oft_blocks
.
device
)
if
self
.
is_kohya
:
block_Q
=
oft_blocks
-
oft_blocks
.
transpose
(
1
,
2
)
# ensure skew-symmetric orthogonal matrix
norm_Q
=
torch
.
norm
(
block_Q
.
flatten
())
new_norm_Q
=
torch
.
clamp
(
norm_Q
,
max
=
self
.
constraint
)
block_Q
=
block_Q
*
((
new_norm_Q
+
1e-8
)
/
(
norm_Q
+
1e-8
))
oft_blocks
=
torch
.
matmul
(
eye
+
block_Q
,
(
eye
-
block_Q
)
.
float
()
.
inverse
())
R
=
oft_blocks
.
to
(
orig_weight
.
device
,
dtype
=
orig_weight
.
dtype
)
R
=
R
*
multiplier
+
torch
.
eye
(
self
.
block_size
,
device
=
orig_weight
.
device
)
# This errors out for MultiheadAttention, might need to be handled up-stream
merged_weight
=
rearrange
(
orig_weight
,
'(k n) ... -> k n ...'
,
k
=
self
.
num_blocks
,
n
=
self
.
block_size
)
...
...
@@ -72,26 +80,3 @@ class NetworkModuleOFT(network.NetworkModule):
updown
=
merged_weight
.
to
(
orig_weight
.
device
,
dtype
=
orig_weight
.
dtype
)
-
orig_weight
output_shape
=
orig_weight
.
shape
return
self
.
finalize_updown
(
updown
,
orig_weight
,
output_shape
)
def
calc_updown
(
self
,
orig_weight
):
# if alpha is a very small number as in coft, calc_scale() will return a almost zero number so we ignore it
multiplier
=
self
.
multiplier
()
return
self
.
calc_updown_kb
(
orig_weight
,
multiplier
)
# override to remove the multiplier/scale factor; it's already multiplied in get_weight
def
finalize_updown
(
self
,
updown
,
orig_weight
,
output_shape
,
ex_bias
=
None
):
if
self
.
bias
is
not
None
:
updown
=
updown
.
reshape
(
self
.
bias
.
shape
)
updown
+=
self
.
bias
.
to
(
orig_weight
.
device
,
dtype
=
orig_weight
.
dtype
)
updown
=
updown
.
reshape
(
output_shape
)
if
len
(
output_shape
)
==
4
:
updown
=
updown
.
reshape
(
output_shape
)
if
orig_weight
.
size
()
.
numel
()
==
updown
.
size
()
.
numel
():
updown
=
updown
.
reshape
(
orig_weight
.
shape
)
if
ex_bias
is
not
None
:
ex_bias
=
ex_bias
*
self
.
multiplier
()
return
updown
,
ex_bias
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment