nsarrazin HF Staff commited on
Commit
03e84b5
·
unverified ·
1 Parent(s): 296108d

Update scripts & documentations to reflect new prod setup (#1107)

Browse files
.env.template DELETED
@@ -1,298 +0,0 @@
1
- # template used in production for HuggingChat.
2
-
3
- MODELS=`[
4
- {
5
- "name" : "CohereForAI/c4ai-command-r-plus",
6
- "tokenizer": "Xenova/c4ai-command-r-v01-tokenizer",
7
- "description": "Command R+ is Cohere's latest LLM and is the first open weight model to beat GPT4 in the Chatbot Arena!",
8
- "modelUrl": "https://huggingface.co/CohereForAI/c4ai-command-r-plus",
9
- "websiteUrl": "https://docs.cohere.com/docs/command-r-plus",
10
- "logoUrl": "https://huggingface.co/datasets/huggingchat/models-logo/resolve/main/cohere-logo.png",
11
- "parameters": {
12
- "stop": ["<|END_OF_TURN_TOKEN|>"],
13
- "truncate" : 28672,
14
- "max_new_tokens" : 4096,
15
- "temperature" : 0.3
16
- },
17
- "promptExamples" : [
18
- {
19
- "title": "Write an email from bullet list",
20
- "prompt": "As a restaurant owner, write a professional email to the supplier to get these products every week: \n\n- Wine (x10)\n- Eggs (x24)\n- Bread (x12)"
21
- }, {
22
- "title": "Code a snake game",
23
- "prompt": "Code a basic snake game in python, give explanations for each step."
24
- }, {
25
- "title": "Assist in a task",
26
- "prompt": "How do I make a delicious lemon cheesecake?"
27
- }
28
- ]
29
- },
30
- {
31
- "name" : "meta-llama/Meta-Llama-3-70B-Instruct",
32
- "description": "Generation over generation, Meta Llama 3 demonstrates state-of-the-art performance on a wide range of industry benchmarks and offers new capabilities, including improved reasoning.",
33
- "logoUrl": "https://huggingface.co/datasets/huggingchat/models-logo/resolve/main/meta-logo.png",
34
- "modelUrl": "https://huggingface.co/meta-llama/Meta-Llama-3-70B-Instruct",
35
- "websiteUrl": "https://llama.meta.com/llama3/",
36
- "tokenizer" : "philschmid/meta-llama-3-tokenizer",
37
- "promptExamples" : [
38
- {
39
- "title": "Write an email from bullet list",
40
- "prompt": "As a restaurant owner, write a professional email to the supplier to get these products every week: \n\n- Wine (x10)\n- Eggs (x24)\n- Bread (x12)"
41
- }, {
42
- "title": "Code a snake game",
43
- "prompt": "Code a basic snake game in python, give explanations for each step."
44
- }, {
45
- "title": "Assist in a task",
46
- "prompt": "How do I make a delicious lemon cheesecake?"
47
- }
48
- ],
49
- "parameters": {
50
- "stop": ["<|eot_id|>"],
51
- "truncate": 6144,
52
- "max_new_tokens": 2047
53
- }
54
- },
55
- {
56
- "name" : "HuggingFaceH4/zephyr-orpo-141b-A35b-v0.1",
57
- "tokenizer": "HuggingFaceH4/zephyr-orpo-141b-A35b-v0.1",
58
- "description": "Zephyr 141B-A35B is a fine-tuned version of Mistral 8x22B, trained using ORPO, a novel alignment algorithm.",
59
- "modelUrl": "https://huggingface.co/HuggingFaceH4/zephyr-orpo-141b-A35b-v0.1",
60
- "websiteUrl": "https://huggingface.co/HuggingFaceH4/zephyr-orpo-141b-A35b-v0.1",
61
- "logoUrl": "https://huggingface.co/datasets/huggingchat/models-logo/resolve/main/zephyr-logo.png",
62
- "parameters": {
63
- "truncate" : 24576,
64
- "max_new_tokens" : 8192,
65
- },
66
- "preprompt" : "You are Zephyr, an assistant developed by KAIST AI, Argilla, and Hugging Face. You should give concise responses to very simple questions, but provide thorough responses to more complex and open-ended questions. You are happy to help with writing, analysis, question answering, math, coding, and all sorts of other tasks.",
67
- "promptExamples" : [
68
- {
69
- "title": "Write a poem",
70
- "prompt": "Write a poem to help me remember the first 10 elements on the periodic table, giving each element its own line."
71
- }, {
72
- "title": "Code a snake game",
73
- "prompt": "Code a basic snake game in python, give explanations for each step."
74
- }, {
75
- "title": "Assist in a task",
76
- "prompt": "How do I make a delicious lemon cheesecake?"
77
- }
78
- ]
79
- },
80
- {
81
- "name" : "mistralai/Mixtral-8x7B-Instruct-v0.1",
82
- "description" : "The latest MoE model from Mistral AI! 8x7B and outperforms Llama 2 70B in most benchmarks.",
83
- "logoUrl": "https://huggingface.co/datasets/huggingchat/models-logo/resolve/main/mistral-logo.png",
84
- "websiteUrl" : "https://mistral.ai/news/mixtral-of-experts/",
85
- "modelUrl": "https://huggingface.co/mistralai/Mixtral-8x7B-Instruct-v0.1",
86
- "tokenizer": "mistralai/Mixtral-8x7B-Instruct-v0.1",
87
- "preprompt" : "",
88
- "chatPromptTemplate": "<s> {{#each messages}}{{#ifUser}}[INST]{{#if @first}}{{#if @root.preprompt}}{{@root.preprompt}}\n{{/if}}{{/if}} {{content}} [/INST]{{/ifUser}}{{#ifAssistant}} {{content}}</s> {{/ifAssistant}}{{/each}}",
89
- "parameters" : {
90
- "temperature" : 0.6,
91
- "top_p" : 0.95,
92
- "repetition_penalty" : 1.2,
93
- "top_k" : 50,
94
- "truncate" : 24576,
95
- "max_new_tokens" : 8192,
96
- "stop" : ["</s>"]
97
- },
98
- "promptExamples" : [
99
- {
100
- "title": "Write an email from bullet list",
101
- "prompt": "As a restaurant owner, write a professional email to the supplier to get these products every week: \n\n- Wine (x10)\n- Eggs (x24)\n- Bread (x12)"
102
- }, {
103
- "title": "Code a snake game",
104
- "prompt": "Code a basic snake game in python, give explanations for each step."
105
- }, {
106
- "title": "Assist in a task",
107
- "prompt": "How do I make a delicious lemon cheesecake?"
108
- }
109
- ]
110
- },
111
- {
112
- "name" : "NousResearch/Nous-Hermes-2-Mixtral-8x7B-DPO",
113
- "description" : "Nous Hermes 2 Mixtral 8x7B DPO is the new flagship Nous Research model trained over the Mixtral 8x7B MoE LLM.",
114
- "logoUrl": "https://huggingface.co/datasets/huggingchat/models-logo/resolve/main/nous-logo.png",
115
- "websiteUrl" : "https://nousresearch.com/",
116
- "modelUrl": "https://huggingface.co/NousResearch/Nous-Hermes-2-Mixtral-8x7B-DPO",
117
- "tokenizer": "NousResearch/Nous-Hermes-2-Mixtral-8x7B-DPO",
118
- "chatPromptTemplate" : "{{#if @root.preprompt}}<|im_start|>system\n{{@root.preprompt}}<|im_end|>\n{{/if}}{{#each messages}}{{#ifUser}}<|im_start|>user\n{{content}}<|im_end|>\n<|im_start|>assistant\n{{/ifUser}}{{#ifAssistant}}{{content}}<|im_end|>\n{{/ifAssistant}}{{/each}}",
119
- "promptExamples": [
120
- {
121
- "title": "Write an email from bullet list",
122
- "prompt": "As a restaurant owner, write a professional email to the supplier to get these products every week: \n\n- Wine (x10)\n- Eggs (x24)\n- Bread (x12)"
123
- }, {
124
- "title": "Code a snake game",
125
- "prompt": "Code a basic snake game in python, give explanations for each step."
126
- }, {
127
- "title": "Assist in a task",
128
- "prompt": "How do I make a delicious lemon cheesecake?"
129
- }
130
- ],
131
- "parameters": {
132
- "temperature": 0.7,
133
- "top_p": 0.95,
134
- "repetition_penalty": 1,
135
- "top_k": 50,
136
- "truncate": 24576,
137
- "max_new_tokens": 2048,
138
- "stop": ["<|im_end|>"]
139
- }
140
- },
141
- {
142
- "name" : "google/gemma-1.1-7b-it",
143
- "description": "Gemma 7B 1.1 is the latest release in the Gemma family of lightweight models built by Google, trained using a novel RLHF method.",
144
- "websiteUrl" : "https://blog.google/technology/developers/gemma-open-models/",
145
- "logoUrl": "https://huggingface.co/datasets/huggingchat/models-logo/resolve/main/google-logo.png",
146
- "modelUrl": "https://huggingface.co/google/gemma-1.1-7b-it",
147
- "preprompt": "",
148
- "chatPromptTemplate" : "{{#each messages}}{{#ifUser}}<start_of_turn>user\n{{#if @first}}{{#if @root.preprompt}}{{@root.preprompt}}\n{{/if}}{{/if}}{{content}}<end_of_turn>\n<start_of_turn>model\n{{/ifUser}}{{#ifAssistant}}{{content}}<end_of_turn>\n{{/ifAssistant}}{{/each}}",
149
- "promptExamples": [
150
- {
151
- "title": "Write an email from bullet list",
152
- "prompt": "As a restaurant owner, write a professional email to the supplier to get these products every week: \n\n- Wine (x10)\n- Eggs (x24)\n- Bread (x12)"
153
- }, {
154
- "title": "Code a snake game",
155
- "prompt": "Code a basic snake game in python, give explanations for each step."
156
- }, {
157
- "title": "Assist in a task",
158
- "prompt": "How do I make a delicious lemon cheesecake?"
159
- }
160
- ],
161
- "parameters": {
162
- "do_sample": true,
163
- "truncate": 7168,
164
- "max_new_tokens": 1024,
165
- "stop" : ["<end_of_turn>"]
166
- }
167
- },
168
-
169
- {
170
- "name": "mistralai/Mistral-7B-Instruct-v0.2",
171
- "displayName": "mistralai/Mistral-7B-Instruct-v0.2",
172
- "description": "Mistral 7B is a new Apache 2.0 model, released by Mistral AI that outperforms Llama2 13B in benchmarks.",
173
- "logoUrl": "https://huggingface.co/datasets/huggingchat/models-logo/resolve/main/mistral-logo.png",
174
- "websiteUrl": "https://mistral.ai/news/announcing-mistral-7b/",
175
- "modelUrl": "https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.2",
176
- "tokenizer": "mistralai/Mistral-7B-Instruct-v0.2",
177
- "preprompt": "",
178
- "chatPromptTemplate" : "<s>{{#each messages}}{{#ifUser}}[INST] {{#if @first}}{{#if @root.preprompt}}{{@root.preprompt}}\n{{/if}}{{/if}}{{content}} [/INST]{{/ifUser}}{{#ifAssistant}}{{content}}</s>{{/ifAssistant}}{{/each}}",
179
- "parameters": {
180
- "temperature": 0.3,
181
- "top_p": 0.95,
182
- "repetition_penalty": 1.2,
183
- "top_k": 50,
184
- "truncate": 3072,
185
- "max_new_tokens": 1024,
186
- "stop": ["</s>"]
187
- },
188
- "promptExamples": [
189
- {
190
- "title": "Write an email from bullet list",
191
- "prompt": "As a restaurant owner, write a professional email to the supplier to get these products every week: \n\n- Wine (x10)\n- Eggs (x24)\n- Bread (x12)"
192
- }, {
193
- "title": "Code a snake game",
194
- "prompt": "Code a basic snake game in python, give explanations for each step."
195
- }, {
196
- "title": "Assist in a task",
197
- "prompt": "How do I make a delicious lemon cheesecake?"
198
- }
199
- ]
200
- },
201
- {
202
- "name": "microsoft/Phi-3-mini-4k-instruct",
203
- "tokenizer": "microsoft/Phi-3-mini-4k-instruct",
204
- "description" : "Phi-3 Mini-4K-Instruct is a 3.8B parameters, lightweight, state-of-the-art open model built upon datasets used for Phi-2.",
205
- "logoUrl": "https://huggingface.co/datasets/huggingchat/models-logo/resolve/main/microsoft-logo.png",
206
- "modelUrl": "https://huggingface.co/microsoft/Phi-3-mini-4k-instruct",
207
- "websiteUrl": "https://azure.microsoft.com/en-us/blog/introducing-phi-3-redefining-whats-possible-with-slms/",
208
- "preprompt": "",
209
- "chatPromptTemplate": "<s>{{preprompt}}{{#each messages}}{{#ifUser}}<|user|>\n{{content}}<|end|>\n<|assistant|>\n{{/ifUser}}{{#ifAssistant}}{{content}}<|end|>\n{{/ifAssistant}}{{/each}}",
210
- "parameters": {
211
- "stop": ["<|end|>", "<|endoftext|>", "<|assistant|>"],
212
- "max_new_tokens": 1024,
213
- "truncate": 3071
214
- },
215
- "promptExamples": [
216
- {
217
- "title": "Write an email from bullet list",
218
- "prompt": "As a restaurant owner, write a professional email to the supplier to get these products every week: \n\n- Wine (x10)\n- Eggs (x24)\n- Bread (x12)"
219
- }, {
220
- "title": "Code a snake game",
221
- "prompt": "Code a basic snake game in python, give explanations for each step."
222
- }, {
223
- "title": "Assist in a task",
224
- "prompt": "How do I make a delicious lemon cheesecake?"
225
- }
226
- ]
227
- },
228
- {
229
- "name": "meta-llama/Meta-Llama-3-8B-Instruct",
230
- "tokenizer" : "philschmid/meta-llama-3-tokenizer",
231
- "parameters": {
232
- "temperature": 0.1,
233
- "stop": ["<|eot_id|>"],
234
- "truncate": 1024,
235
- },
236
- "unlisted": true
237
- }
238
- ]`
239
-
240
- OLD_MODELS=`[
241
- {"name":"bigcode/starcoder"},
242
- {"name":"OpenAssistant/oasst-sft-6-llama-30b-xor"},
243
- {"name":"HuggingFaceH4/zephyr-7b-alpha"},
244
- {"name":"openchat/openchat_3.5"},
245
- {"name":"openchat/openchat-3.5-1210"},
246
- {"name": "tiiuae/falcon-180B-chat"},
247
- {"name": "codellama/CodeLlama-34b-Instruct-hf"},
248
- {"name": "google/gemma-7b-it"},
249
- {"name": "meta-llama/Llama-2-70b-chat-hf"},
250
- {"name": "codellama/CodeLlama-70b-Instruct-hf"},
251
- {"name": "openchat/openchat-3.5-0106"}
252
- ]`
253
-
254
- TASK_MODEL='meta-llama/Meta-Llama-3-8B-Instruct'
255
-
256
- TEXT_EMBEDDING_MODELS = `[
257
- {
258
- "name": "bge-base-en-v1-5-sxa",
259
- "displayName": "bge-base-en-v1-5-sxa",
260
- "chunkCharLength": 512,
261
- "endpoints": [
262
- { "type": "tei",
263
- "url" : "https://huggingchat-tei.hf.space/"
264
- }
265
- ]
266
- }
267
- ]`
268
-
269
-
270
- APP_BASE="/chat"
271
- PUBLIC_ORIGIN=https://huggingface.co
272
- PUBLIC_SHARE_PREFIX=https://hf.co/chat
273
- PUBLIC_ANNOUNCEMENT_BANNERS=`[]`
274
-
275
- PUBLIC_APP_NAME=HuggingChat
276
- PUBLIC_APP_ASSETS=huggingchat
277
- PUBLIC_APP_COLOR=yellow
278
- PUBLIC_APP_DESCRIPTION="Making the community's best AI chat models available to everyone."
279
- PUBLIC_APP_DISCLAIMER_MESSAGE="Disclaimer: AI is an area of active research with known problems such as biased generation and misinformation. Do not use this application for high-stakes decisions or advice."
280
- PUBLIC_APP_DATA_SHARING=0
281
- PUBLIC_APP_DISCLAIMER=1
282
-
283
- PUBLIC_PLAUSIBLE_SCRIPT_URL="/js/script.js"
284
- PUBLIC_APPLE_APP_ID=6476778843
285
- # Not part of the .env but set as other variables in the space
286
- # ADDRESS_HEADER=X-Forwarded-For
287
- # XFF_DEPTH=2
288
-
289
- ENABLE_ASSISTANTS=true
290
- ENABLE_ASSISTANTS_RAG=true
291
- REQUIRE_FEATURED_ASSISTANTS=true
292
- EXPOSE_API=true
293
-
294
- ALTERNATIVE_REDIRECT_URLS=`[
295
- huggingchat://login/callback
296
- ]`
297
-
298
- WEBSEARCH_BLOCKLIST=`["youtube.com", "twitter.com"]`
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
.github/workflows/deploy-release.yml DELETED
@@ -1,44 +0,0 @@
1
- name: Deploy to production spaces
2
- on:
3
- # run this workflow manually from the Actions tab
4
- workflow_dispatch:
5
-
6
- jobs:
7
- update-env:
8
- runs-on: ubuntu-latest
9
- timeout-minutes: 10
10
-
11
- steps:
12
- - uses: actions/checkout@v3
13
- - uses: actions/setup-node@v3
14
- with:
15
- node-version: "20"
16
- cache: "npm"
17
- - run: npm install ci
18
- - name: "Update DOTENV_LOCAL in prod"
19
- env:
20
- HF_TOKEN: ${{ secrets.HF_TOKEN }}
21
- SERPER_API_KEY: ${{ secrets.SERPER_API_KEY }}
22
- OPENID_CONFIG: ${{ secrets.OPENID_CONFIG }}
23
- MONGODB_URL: ${{ secrets.MONGODB_URL }}
24
- HF_DEPLOYMENT_TOKEN: ${{ secrets.HF_DEPLOYMENT_TOKEN }}
25
- WEBHOOK_URL_REPORT_ASSISTANT: ${{ secrets.WEBHOOK_URL_REPORT_ASSISTANT }}
26
- ADMIN_API_SECRET: ${{ secrets.ADMIN_API_SECRET }}
27
- USAGE_LIMITS: ${{ secrets.USAGE_LIMITS }}
28
- MESSAGES_BEFORE_LOGIN: ${{ secrets.MESSAGES_BEFORE_LOGIN }}
29
- run: npm run updateProdEnv
30
- sync-to-hub:
31
- runs-on: ubuntu-latest
32
- steps:
33
- - name: Check large files
34
- uses: ActionsDesk/[email protected]
35
- with:
36
- filesizelimit: 10485760 # this is 10MB so we can sync to HF Spaces
37
- - uses: actions/checkout@v3
38
- with:
39
- fetch-depth: 0
40
- lfs: true
41
- - name: Push to hub
42
- env:
43
- HF_DEPLOYMENT_TOKEN: ${{ secrets.HF_DEPLOYMENT_TOKEN }}
44
- run: git push https://nsarrazin:[email protected]/spaces/huggingchat/chat-ui main
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
.gitignore CHANGED
@@ -11,5 +11,4 @@ SECRET_CONFIG
11
  .idea
12
  !.env.ci
13
  !.env
14
- !.env.template
15
  gcp-*.json
 
11
  .idea
12
  !.env.ci
13
  !.env
 
14
  gcp-*.json
README.md CHANGED
@@ -767,17 +767,10 @@ You can preview the production build with `npm run preview`.
767
 
768
  ## Config changes for HuggingChat
769
 
770
- The config file for HuggingChat is stored in the `.env.template` file at the root of the repository. It is the single source of truth that is used to generate the actual `.env.local` file using our CI/CD pipeline. See [updateProdEnv](https://github.com/huggingface/chat-ui/blob/cdb33a9583f5339ade724db615347393ef48f5cd/scripts/updateProdEnv.ts) for more details.
771
 
772
  > [!TIP]
773
- > If you want to make changes to the model config used in production for HuggingChat, you should do so against `.env.template`.
774
-
775
- We currently use the following secrets for deploying HuggingChat in addition to the `.env.template` above:
776
-
777
- - `MONGODB_URL`
778
- - `HF_TOKEN`
779
- - `OPENID_CONFIG`
780
- - `SERPER_API_KEY`
781
 
782
  ### Running a copy of HuggingChat locally
783
 
@@ -802,7 +795,7 @@ SERPER_API_KEY=<your serper API key from step 3>
802
  MESSAGES_BEFORE_LOGIN=<can be any numerical value, or set to 0 to require login>
803
  ```
804
 
805
- You can then run `npm run updateLocalEnv` in the root of chat-ui. This will create a `.env.local` file which combines the `.env.template` and the `.env.SECRET_CONFIG` file. You can then run `npm run dev` to start your local instance of HuggingChat.
806
 
807
  ### Populate database
808
 
 
767
 
768
  ## Config changes for HuggingChat
769
 
770
+ The config file for HuggingChat is stored in the `chart/env/prod.yaml` file. It is the source of truth for the environment variables used for our CI/CD pipeline. For HuggingChat, as we need to customize the app color, as well as the base path, we build a custom docker image. You can find the workflow here.
771
 
772
  > [!TIP]
773
+ > If you want to make changes to the model config used in production for HuggingChat, you should do so against `chart/env/prod.yaml`.
 
 
 
 
 
 
 
774
 
775
  ### Running a copy of HuggingChat locally
776
 
 
795
  MESSAGES_BEFORE_LOGIN=<can be any numerical value, or set to 0 to require login>
796
  ```
797
 
798
+ You can then run `npm run updateLocalEnv` in the root of chat-ui. This will create a `.env.local` file which combines the `chart/env/prod.yaml` and the `.env.SECRET_CONFIG` file. You can then run `npm run dev` to start your local instance of HuggingChat.
799
 
800
  ### Populate database
801
 
package-lock.json CHANGED
@@ -51,6 +51,7 @@
51
  "@sveltejs/kit": "^1.30.4",
52
  "@tailwindcss/typography": "^0.5.9",
53
  "@types/express": "^4.17.21",
 
54
  "@types/jsdom": "^21.1.1",
55
  "@types/minimist": "^1.2.5",
56
  "@types/parquetjs": "^0.10.3",
@@ -60,6 +61,7 @@
60
  "eslint": "^8.28.0",
61
  "eslint-config-prettier": "^8.5.0",
62
  "eslint-plugin-svelte": "^2.30.0",
 
63
  "minimist": "^1.2.8",
64
  "prettier": "^2.8.0",
65
  "prettier-plugin-svelte": "^2.10.1",
@@ -2075,6 +2077,12 @@
2075
  "integrity": "sha512-D0CFMMtydbJAegzOyHjtiKPLlvnm3iTZyZRSZoLq2mRhDdmLfIWOCYPfQJ4cu2erKghU++QvjcUjp/5h7hESpA==",
2076
  "dev": true
2077
  },
 
 
 
 
 
 
2078
  "node_modules/@types/jsdom": {
2079
  "version": "21.1.1",
2080
  "resolved": "https://registry.npmjs.org/@types/jsdom/-/jsdom-21.1.1.tgz",
 
51
  "@sveltejs/kit": "^1.30.4",
52
  "@tailwindcss/typography": "^0.5.9",
53
  "@types/express": "^4.17.21",
54
+ "@types/js-yaml": "^4.0.9",
55
  "@types/jsdom": "^21.1.1",
56
  "@types/minimist": "^1.2.5",
57
  "@types/parquetjs": "^0.10.3",
 
61
  "eslint": "^8.28.0",
62
  "eslint-config-prettier": "^8.5.0",
63
  "eslint-plugin-svelte": "^2.30.0",
64
+ "js-yaml": "^4.1.0",
65
  "minimist": "^1.2.8",
66
  "prettier": "^2.8.0",
67
  "prettier-plugin-svelte": "^2.10.1",
 
2077
  "integrity": "sha512-D0CFMMtydbJAegzOyHjtiKPLlvnm3iTZyZRSZoLq2mRhDdmLfIWOCYPfQJ4cu2erKghU++QvjcUjp/5h7hESpA==",
2078
  "dev": true
2079
  },
2080
+ "node_modules/@types/js-yaml": {
2081
+ "version": "4.0.9",
2082
+ "resolved": "https://registry.npmjs.org/@types/js-yaml/-/js-yaml-4.0.9.tgz",
2083
+ "integrity": "sha512-k4MGaQl5TGo/iipqb2UDG2UwjXziSWkh0uysQelTlJpX1qGlpUZYm8PnO4DxG1qBomtJUdYJ6qR6xdIah10JLg==",
2084
+ "dev": true
2085
+ },
2086
  "node_modules/@types/jsdom": {
2087
  "version": "21.1.1",
2088
  "resolved": "https://registry.npmjs.org/@types/jsdom/-/jsdom-21.1.1.tgz",
package.json CHANGED
@@ -24,6 +24,7 @@
24
  "@sveltejs/kit": "^1.30.4",
25
  "@tailwindcss/typography": "^0.5.9",
26
  "@types/express": "^4.17.21",
 
27
  "@types/jsdom": "^21.1.1",
28
  "@types/minimist": "^1.2.5",
29
  "@types/parquetjs": "^0.10.3",
@@ -33,6 +34,7 @@
33
  "eslint": "^8.28.0",
34
  "eslint-config-prettier": "^8.5.0",
35
  "eslint-plugin-svelte": "^2.30.0",
 
36
  "minimist": "^1.2.8",
37
  "prettier": "^2.8.0",
38
  "prettier-plugin-svelte": "^2.10.1",
 
24
  "@sveltejs/kit": "^1.30.4",
25
  "@tailwindcss/typography": "^0.5.9",
26
  "@types/express": "^4.17.21",
27
+ "@types/js-yaml": "^4.0.9",
28
  "@types/jsdom": "^21.1.1",
29
  "@types/minimist": "^1.2.5",
30
  "@types/parquetjs": "^0.10.3",
 
34
  "eslint": "^8.28.0",
35
  "eslint-config-prettier": "^8.5.0",
36
  "eslint-plugin-svelte": "^2.30.0",
37
+ "js-yaml": "^4.1.0",
38
  "minimist": "^1.2.8",
39
  "prettier": "^2.8.0",
40
  "prettier-plugin-svelte": "^2.10.1",
scripts/updateLocalEnv.ts CHANGED
@@ -1,17 +1,40 @@
1
  import fs from "fs";
 
2
 
3
- const SECRET_CONFIG = fs.existsSync(".env.SECRET_CONFIG")
4
- ? fs.readFileSync(".env.SECRET_CONFIG", "utf8")
5
- : process.env.SECRET_CONFIG;
 
 
 
 
 
 
 
 
 
 
 
 
 
6
 
7
  if (!SECRET_CONFIG) {
8
- throw new Error(
9
- "SECRET_CONFIG is not defined. Please provide it either in a file or as an environment variable."
10
  );
11
- }
12
 
13
- // Read the content of the file .env.template
14
- const PUBLIC_CONFIG = fs.readFileSync(".env.template", "utf8");
 
 
 
 
 
 
 
 
 
 
15
 
16
  // Prepend the content of the env variable SECRET_CONFIG
17
  const full_config = `${PUBLIC_CONFIG}\n${SECRET_CONFIG}`;
 
1
  import fs from "fs";
2
+ import yaml from "js-yaml";
3
 
4
+ const file = fs.readFileSync("chart/env/prod.yaml", "utf8");
5
+
6
+ // have to do a weird stringify/parse because of some node error
7
+ const prod = JSON.parse(JSON.stringify(yaml.load(file)));
8
+ const vars = prod.envVars as Record<string, string>;
9
+ const secrets = prod.externalSecrets.parameters;
10
+
11
+ let PUBLIC_CONFIG = "";
12
+ Object.entries(vars).forEach(([key, value]) => {
13
+ PUBLIC_CONFIG += `${key}=\`${value}\`\n`;
14
+ });
15
+
16
+ let SECRET_CONFIG =
17
+ (fs.existsSync(".env.SECRET_CONFIG")
18
+ ? fs.readFileSync(".env.SECRET_CONFIG", "utf8")
19
+ : process.env.SECRET_CONFIG) ?? "";
20
 
21
  if (!SECRET_CONFIG) {
22
+ console.log(
23
+ "SECRET_CONFIG is not defined. We will now try to fill in secrets found in the prod environemnt with environment variables."
24
  );
 
25
 
26
+ Object.keys(secrets).forEach((key) => {
27
+ const value = process.env[key];
28
+
29
+ if (!value) {
30
+ throw new Error(
31
+ `Secret ${key} was found in prod.yaml but was not available as an environment variable.`
32
+ );
33
+ }
34
+
35
+ SECRET_CONFIG += `${key}=${value}\n`;
36
+ });
37
+ }
38
 
39
  // Prepend the content of the env variable SECRET_CONFIG
40
  const full_config = `${PUBLIC_CONFIG}\n${SECRET_CONFIG}`;
scripts/updateProdEnv.ts DELETED
@@ -1,41 +0,0 @@
1
- import fs from "fs";
2
-
3
- const HF_DEPLOYMENT_TOKEN = process.env.HF_DEPLOYMENT_TOKEN; // token used for pushing to hub
4
-
5
- const SERPER_API_KEY = process.env.SERPER_API_KEY;
6
- const OPENID_CONFIG = process.env.OPENID_CONFIG;
7
- const MONGODB_URL = process.env.MONGODB_URL;
8
- const HF_TOKEN = process.env.HF_TOKEN ?? process.env.HF_ACCESS_TOKEN; // token used for API requests in prod
9
- const WEBHOOK_URL_REPORT_ASSISTANT = process.env.WEBHOOK_URL_REPORT_ASSISTANT; // slack webhook url used to get "report assistant" events
10
- const ADMIN_API_SECRET = process.env.ADMIN_API_SECRET;
11
- const USAGE_LIMITS = process.env.USAGE_LIMITS;
12
- const MESSAGES_BEFORE_LOGIN = process.env.MESSAGES_BEFORE_LOGIN;
13
-
14
- // Read the content of the file .env.template
15
- const PUBLIC_CONFIG = fs.readFileSync(".env.template", "utf8");
16
-
17
- // Prepend the content of the env variable SECRET_CONFIG
18
- const full_config = `${PUBLIC_CONFIG}
19
- MONGODB_URL=${MONGODB_URL}
20
- OPENID_CONFIG=${OPENID_CONFIG}
21
- SERPER_API_KEY=${SERPER_API_KEY}
22
- HF_TOKEN=${HF_TOKEN}
23
- WEBHOOK_URL_REPORT_ASSISTANT=${WEBHOOK_URL_REPORT_ASSISTANT}
24
- ADMIN_API_SECRET=${ADMIN_API_SECRET}
25
- USAGE_LIMITS=${USAGE_LIMITS}
26
- MESSAGES_BEFORE_LOGIN=${MESSAGES_BEFORE_LOGIN}
27
- `;
28
-
29
- // Make an HTTP POST request to add the space secrets
30
- fetch(`https://huggingface.co/api/spaces/huggingchat/chat-ui/secrets`, {
31
- method: "POST",
32
- body: JSON.stringify({
33
- key: "DOTENV_LOCAL",
34
- value: full_config,
35
- description: `Env variable for HuggingChat. Last updated ${new Date().toISOString()}`,
36
- }),
37
- headers: {
38
- Authorization: `Bearer ${HF_DEPLOYMENT_TOKEN}`,
39
- "Content-Type": "application/json",
40
- },
41
- });