dacorvo HF Staff commited on
Commit
c564534
·
verified ·
1 Parent(s): 89020e2

Add phi4 cached configurations

Browse files
Files changed (1) hide show
  1. inference-cache-config/phi4.json +30 -0
inference-cache-config/phi4.json ADDED
@@ -0,0 +1,30 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "microsoft/Phi-3-mini-4k-instruct": [
3
+ {
4
+ "batch_size": 1,
5
+ "sequence_length": 4096,
6
+ "num_cores": 2,
7
+ "auto_cast_type": "bf16"
8
+ },
9
+ {
10
+ "batch_size": 4,
11
+ "sequence_length": 4096,
12
+ "num_cores": 2,
13
+ "auto_cast_type": "bf16"
14
+ }
15
+ ],
16
+ "microsoft/phi-4": [
17
+ {
18
+ "batch_size": 1,
19
+ "sequence_length": 4096,
20
+ "num_cores": 10,
21
+ "auto_cast_type": "bf16"
22
+ },
23
+ {
24
+ "batch_size": 16,
25
+ "sequence_length": 4096,
26
+ "num_cores": 10,
27
+ "auto_cast_type": "bf16"
28
+ }
29
+ ]
30
+ }