feat: mime types on tools, show upload btn on demand (#1204)

* feat: mime types on tools, show upload btn on demand

* feat: enable and disable all tools button

* feat: minor tools ui changes and nits

* Only paste or drop the allowed MIME types (#1222)

* Only allow specific MIME types when pasting

* lint

* Simplify upload button show logic

Since we already take into account if the model is multimodal above

* Add mime type check to file dropzone

* Make error clearer

* enable/disable same button

* cs

* rm unused

---------

Co-authored-by: Nathan Sarrazin <[email protected]>
Co-authored-by: Victor Mustar <[email protected]>

src/lib/components/ToolsMenu.svelte

@@ -16,6 +16,13 @@
 	$: activeToolCount = $page.data.tools.filter(
 		(tool: ToolFront) => $settings?.tools?.[tool.name] ?? tool.isOnByDefault
 	).length;
+
+	function setAllTools(value: boolean) {
+		settings.instantSet({
+			tools: Object.fromEntries($page.data.tools.map((tool: ToolFront) => [tool.name, value])),
+		});
+	}
+	$: allToolsEnabled = activeToolCount === $page.data.tools.length;
 </script>
 
 <details
@@ -39,7 +46,7 @@
 		class="absolute bottom-10 h-max w-max select-none items-center gap-1 rounded-lg border bg-white p-0.5 shadow-sm dark:border-gray-800 dark:bg-gray-900"
 	>
 		<div class="grid grid-cols-2 gap-x-6 gap-y-1 p-3">
-			<div class="col-span-2 mb-1 flex items-center gap-1.5 text-sm text-gray-500">
+			<div class="col-span-2 flex items-center gap-1.5 text-sm text-gray-500">
 				Available tools
 				{#if isHuggingChat}
 					<a
@@ -49,6 +56,16 @@
 						><CarbonInformation class="text-xs" /></a
 					>
 				{/if}
+				<button
+					class="ml-auto text-xs underline"
+					on:click|stopPropagation={() => setAllTools(!allToolsEnabled)}
+				>
+					{#if allToolsEnabled}
+						Disable all
+					{:else}
+						Enable all
+					{/if}
+				</button>
 			</div>
 			{#each $page.data.tools as tool}
 				{@const isChecked = $settings?.tools?.[tool.name] ?? tool.isOnByDefault}

src/lib/components/UploadBtn.svelte

@@ -3,6 +3,7 @@
 
 	export let classNames = "";
 	export let files: File[];
+	export let mimeTypes: string[];
 
 	/**
 	 * Due to a bug with Svelte, we cannot use bind:files with multiple
@@ -22,7 +23,7 @@
 		class="absolute w-full cursor-pointer opacity-0"
 		type="file"
 		on:change={onFileChange}
-		accept="*/*"
+		accept={mimeTypes.join(",")}
 	/>
 	<CarbonUpload class="mr-2 text-xxs" /> Upload file
 </button>

src/lib/components/chat/ChatWindow.svelte

@@ -33,6 +33,8 @@
 	import ChatIntroduction from "./ChatIntroduction.svelte";
 	import { useConvTreeStore } from "$lib/stores/convTree";
 	import UploadedFile from "./UploadedFile.svelte";
+	import { useSettingsStore } from "$lib/stores/settings";
+	import type { ToolFront } from "$lib/types/Tool";
 
 	export let messages: Message[] = [];
 	export let loading = false;
@@ -93,7 +95,17 @@
 		const pastedFiles = Array.from(e.clipboardData.files);
 		if (pastedFiles.length !== 0) {
 			e.preventDefault();
-			files = [...files, ...pastedFiles];
+
+			// filter based on activeMimeTypes, including wildcards
+			const filteredFiles = pastedFiles.filter((file) => {
+				return activeMimeTypes.some((mimeType: string) => {
+					const [type, subtype] = mimeType.split("/");
+					const [fileType, fileSubtype] = file.type.split("/");
+					return type === fileType && (subtype === "*" || fileSubtype === subtype);
+				});
+			});
+
+			files = [...files, ...filteredFiles];
 		}
 	};
 
@@ -138,6 +150,17 @@
 	$: if (lastMessage && lastMessage.from === "user") {
 		scrollToBottom();
 	}
+
+	const settings = useSettingsStore();
+
+	// active tools are all the checked tools, either from settings or on by default
+	$: activeTools = $page.data.tools.filter(
+		(tool: ToolFront) => $settings?.tools?.[tool.name] ?? tool.isOnByDefault
+	);
+	$: activeMimeTypes = [
+		...activeTools.flatMap((tool: ToolFront) => tool.mimeTypes ?? []),
+		...(currentModel.multimodal ? ["image/*"] : []),
+	];
 </script>
 
 <div class="relative min-h-0 min-w-0">
@@ -287,8 +310,8 @@
 					/>
 				{:else}
 					<div class="ml-auto gap-2">
-						{#if currentModel.multimodal || currentModel.tools}
-							<UploadBtn bind:files classNames="ml-auto" />
+						{#if activeMimeTypes.length > 0}
+							<UploadBtn bind:files mimeTypes={activeMimeTypes} classNames="ml-auto" />
 						{/if}
 						{#if messages && lastMessage && lastMessage.interrupted && !isReadOnly}
 							<ContinueBtn
@@ -314,12 +337,8 @@
 				class="relative flex w-full max-w-4xl flex-1 items-center rounded-xl border bg-gray-100 focus-within:border-gray-300 dark:border-gray-600 dark:bg-gray-700 dark:focus-within:border-gray-500
 			{isReadOnly ? 'opacity-30' : ''}"
 			>
-				{#if onDrag && (currentModel.multimodal || currentModel.tools)}
-					<FileDropzone
-						bind:files
-						bind:onDrag
-						onlyImages={currentModel.multimodal && !currentModel.tools}
-					/>
+				{#if onDrag && activeMimeTypes.length > 0}
+					<FileDropzone bind:files bind:onDrag mimeTypes={activeMimeTypes} />
 				{:else}
 					<div class="flex w-full flex-1 border-none bg-transparent">
 						{#if lastIsError}

src/lib/components/chat/FileDropzone.svelte

@@ -4,7 +4,7 @@
 	// import EosIconsLoading from "~icons/eos-icons/loading";
 
 	export let files: File[];
-	export let onlyImages: boolean = false;
+	export let mimeTypes: string[] = [];
 
 	let file_error_message = "";
 	let errorTimeout: ReturnType<typeof setTimeout>;
@@ -24,11 +24,19 @@
 			if (event.dataTransfer.items[0].kind === "file") {
 				const file = event.dataTransfer.items[0].getAsFile();
 				if (file) {
-					if (!event.dataTransfer.items[0].type.startsWith("image") && onlyImages) {
-						setErrorMsg("Only images are supported");
+					// check if the file matches the mimeTypes
+					if (
+						!mimeTypes.some((mimeType: string) => {
+							const [type, subtype] = mimeType.split("/");
+							const [fileType, fileSubtype] = file.type.split("/");
+							return type === fileType && (subtype === "*" || fileSubtype === subtype);
+						})
+					) {
+						setErrorMsg(`File type not supported. Only allowed: ${mimeTypes.join(", ")}`);
 						files = [];
 						return;
 					}
+
 					// if file is bigger than 10MB abort
 					if (file.size > 10 * 1024 * 1024) {
 						setErrorMsg("Image is too big. (2MB max)");
@@ -89,7 +97,7 @@
 			class="mb-3 mt-1.5 text-sm text-gray-500 dark:text-gray-400"
 			class:opacity-0={file_error_message}
 		>
-			Drag and drop <span class="font-semibold">one {onlyImages ? "image" : "file"}</span> here
+			Drag and drop <span class="font-semibold">one file</span> here
 		</p>
 	</div>
 </div>

src/lib/server/tools/documentParser.ts

@@ -10,6 +10,7 @@ const documentParser: BackendTool = {
 	displayName: "Document Parser",
 	description: "Use this tool to parse any document and get its content in markdown format.",
 	isOnByDefault: true,
+	mimeTypes: ["application/*", "text/*"],
 	parameterDefinitions: {
 		fileMessageIndex: {
 			description: "Index of the message containing the document file to parse",

src/lib/server/tools/images/editing.ts

@@ -18,6 +18,7 @@ const imageEditing: BackendTool = {
 	displayName: "Image Editing",
 	description: "Use this tool to edit an image from a prompt.",
 	isOnByDefault: true,
+	mimeTypes: ["image/*"],
 	parameterDefinitions: {
 		prompt: {
 			description:

src/lib/types/Tool.ts

@@ -15,6 +15,8 @@ export interface Tool {
 	name: string;
 	displayName?: string;
 	description: string;
+	/** List of mime types that tool accepts */
+	mimeTypes?: string[];
 	parameterDefinitions: Record<string, ToolInput>;
 	spec?: string;
 	isOnByDefault?: true; // will it be toggled if the user hasn't tweaked it in settings ?
@@ -24,7 +26,7 @@ export interface Tool {
 
 export type ToolFront = Pick<
 	Tool,
-	"name" | "displayName" | "description" | "isOnByDefault" | "isLocked"
+	"name" | "displayName" | "description" | "isOnByDefault" | "isLocked" | "mimeTypes"
 > & { timeToUseMS?: number };
 
 export enum ToolResultStatus {

src/routes/+layout.server.ts

@@ -173,6 +173,7 @@ export const load: LayoutServerLoad = async ({ locals, depends }) => {
 				name: tool.name,
 				displayName: tool.displayName,
 				description: tool.description,
+				mimeTypes: tool.mimeTypes,
 				isOnByDefault: tool.isOnByDefault,
 				isLocked: tool.isLocked,
 				timeToUseMS: