From 4939d6871b033894098f5aa4566e3339b6125357 Mon Sep 17 00:00:00 2001 From: KarlLee830 <61072264+KarlLee830@users.noreply.github.com> Date: Tue, 18 Feb 2025 23:40:25 +0800 Subject: [PATCH 01/37] i18n: Update Chinese Translation --- src/lib/i18n/locales/zh-CN/translation.json | 60 ++++++++++----------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/src/lib/i18n/locales/zh-CN/translation.json b/src/lib/i18n/locales/zh-CN/translation.json index 4e26eb022..3b7dfb05d 100644 --- a/src/lib/i18n/locales/zh-CN/translation.json +++ b/src/lib/i18n/locales/zh-CN/translation.json @@ -20,7 +20,7 @@ "Account Activation Pending": "账号待激活", "Accurate information": "提供的信息很准确", "Actions": "自动化", - "Activate": "", + "Activate": "激活", "Activate this command by typing \"/{{COMMAND}}\" to chat input.": "通过输入 \"/{{COMMAND}}\" 激活此命令", "Active Users": "当前在线用户", "Add": "添加", @@ -100,7 +100,7 @@ "Audio": "语音", "August": "八月", "Authenticate": "认证", - "Authentication": "", + "Authentication": "身份验证", "Auto-Copy Response to Clipboard": "自动复制回复到剪贴板", "Auto-playback response": "自动念出回复内容", "Autocomplete Generation": "输入框内容猜测补全", @@ -167,7 +167,7 @@ "Click here to": "点击", "Click here to download user import template file.": "点击此处下载用户导入所需的模板文件。", "Click here to learn more about faster-whisper and see the available models.": "点击此处了解更多关于faster-whisper的信息，并查看可用的模型。", - "Click here to see available models.": "单击此处查看可用型号。", + "Click here to see available models.": "单击此处查看可用模型。", "Click here to select": "点击这里选择", "Click here to select a csv file.": "点击此处选择 csv 文件。", "Click here to select a py file.": "点击此处选择 py 文件。", @@ -180,12 +180,12 @@ "Clone of {{TITLE}}": "{{TITLE}} 的副本", "Close": "关闭", "Code execution": "代码执行", - "Code Execution": "", - "Code Execution Engine": "", + "Code Execution": "代码执行", + "Code Execution Engine": "代码执行引擎", "Code formatted successfully": "代码格式化成功", "Code Interpreter": "代码解释器", "Code Interpreter Engine": "代码解释引擎", - "Code Interpreter Prompt Template": "代码解释器提示模板", + "Code Interpreter Prompt Template": "代码解释器提示词模板", "Collection": "文件集", "Color": "颜色", "ComfyUI": "ComfyUI", @@ -202,7 +202,7 @@ "Confirm Password": "确认密码", "Confirm your action": "确定吗？", "Confirm your new password": "确认新密码", - "Connect to your own OpenAI compatible API endpoints.": "连接到您自己的 OpenAI 兼容 API 端点。", + "Connect to your own OpenAI compatible API endpoints.": "连接到你自己的与 OpenAI 兼容的 API 接口端点。", "Connections": "外部连接", "Constrains effort on reasoning for reasoning models. Only applicable to reasoning models from specific providers that support reasoning effort. (Default: medium)": "限制推理模型的推理努力。仅适用于支持推理努力的特定提供商的推理模型。（默认值：中等）", "Contact Admin for WebUI Access": "请联系管理员以获取访问权限", @@ -214,7 +214,7 @@ "Continue with Email": "使用邮箱登录", "Continue with LDAP": "使用 LDAP 登录", "Control how message text is split for TTS requests. 'Punctuation' splits into sentences, 'paragraphs' splits into paragraphs, and 'none' keeps the message as a single string.": "控制消息文本如何拆分以用于 TTS 请求。“Punctuation”拆分为句子，“paragraphs”拆分为段落，“none”将消息保留为单个字符串。", - "Control the repetition of token sequences in the generated text. A higher value (e.g., 1.5) will penalize repetitions more strongly, while a lower value (e.g., 1.1) will be more lenient. At 1, it is disabled. (Default: 1.1)": "", + "Control the repetition of token sequences in the generated text. A higher value (e.g., 1.5) will penalize repetitions more strongly, while a lower value (e.g., 1.1) will be more lenient. At 1, it is disabled. (Default: 1.1)": "控制生成文本中 Token 的重复。较高的值（例如 1.5）会更强烈地惩罚重复，而较低的值（例如 1.1）则更宽松。设置为 1 时，此功能被禁用。（默认值：1.1）", "Controls": "对话高级设置", "Controls the balance between coherence and diversity of the output. A lower value will result in more focused and coherent text. (Default: 5.0)": "控制输出的连贯性和多样性之间的平衡。较低的值将导致更集中和连贯的文本。（默认值：5.0）", "Copied": "已复制", @@ -270,7 +270,7 @@ "Delete folder?": "删除分组？", "Delete function?": "删除函数？", "Delete Message": "删除消息", - "Delete message?": "", + "Delete message?": "删除消息？", "Delete prompt?": "删除提示词？", "delete this link": "此处删除这个链接", "Delete tool?": "删除工具？", @@ -282,14 +282,14 @@ "Description": "描述", "Didn't fully follow instructions": "没有完全遵照指示", "Direct Connections": "直接连接", - "Direct Connections allow users to connect to their own OpenAI compatible API endpoints.": "直接连接允许用户连接到他们自己的与 OpenAI 兼容的 API 端点。", + "Direct Connections allow users to connect to their own OpenAI compatible API endpoints.": "直接连接Direct Connections 功能 允许用户连接至其自有的、兼容 OpenAI 的 API 端点。", "Direct Connections settings updated": "直接连接设置已更新", "Disabled": "禁用", "Discover a function": "发现更多函数", "Discover a model": "发现更多模型", "Discover a prompt": "发现更多提示词", "Discover a tool": "发现更多工具", - "Discover how to use Open WebUI and seek support from the community.": "", + "Discover how to use Open WebUI and seek support from the community.": "了解如何使用 Open WebUI 并寻求社区支持。", "Discover wonders": "发现奇迹", "Discover, download, and explore custom functions": "发现、下载并探索更多函数", "Discover, download, and explore custom prompts": "发现、下载并探索更多自定义提示词", @@ -314,7 +314,7 @@ "Don't like the style": "不喜欢这个文风", "Done": "完成", "Download": "下载", - "Download as SVG": "", + "Download as SVG": "下载为 SVG", "Download canceled": "下载已取消", "Download Database": "下载数据库", "Drag and drop a file to upload or select a file to view": "拖动文件上传或选择文件查看", @@ -370,7 +370,7 @@ "Enter Chunk Overlap": "输入块重叠 (Chunk Overlap)", "Enter Chunk Size": "输入块大小 (Chunk Size)", "Enter description": "输入简介描述", - "Enter domains separated by commas (e.g., example.com,site.org)": "输入以逗号分隔的域名（例如：example.com,site.org）", + "Enter domains separated by commas (e.g., example.com,site.org)": "输入以逗号分隔的域名（例如：example.com、site.org）", "Enter Exa API Key": "输入 Exa API 密钥", "Enter Github Raw URL": "输入 Github Raw 地址", "Enter Google PSE API Key": "输入 Google PSE API 密钥", @@ -455,7 +455,7 @@ "Failed to save models configuration": "无法保存模型配置", "Failed to update settings": "无法更新设置", "Failed to upload file.": "上传文件失败", - "Features": "", + "Features": "功能", "Features Permissions": "功能权限", "February": "二月", "Feedback History": "反馈历史", @@ -485,7 +485,7 @@ "Form": "手动创建", "Format your variables using brackets like this:": "使用括号格式化你的变量，如下所示：", "Frequency Penalty": "频率惩罚", - "Full Context Mode": "", + "Full Context Mode": "完整上下文模式", "Function": "函数", "Function Calling": "函数调用 (Function Calling)", "Function created successfully": "函数创建成功", @@ -601,7 +601,7 @@ "Leave empty to include all models or select specific models": "留空表示包含所有模型或请选择模型", "Leave empty to use the default prompt, or enter a custom prompt": "留空以使用默认提示词，或输入自定义提示词。", "Leave model field empty to use the default model.": "将模型字段留空以使用默认模型。", - "License": "", + "License": "授权", "Light": "浅色", "Listening...": "正在倾听...", "Llama.cpp": "Llama.cpp", @@ -761,7 +761,7 @@ "Playground": "AI 对话游乐场", "Please carefully review the following warnings:": "请仔细阅读以下警告信息：", "Please do not close the settings page while loading the model.": "加载模型时请不要关闭设置页面。", - "Please enter a prompt": "请输出一个 Prompt", + "Please enter a prompt": "请输入一个 Prompt", "Please fill in all fields.": "请填写所有字段。", "Please select a model first.": "请先选择一个模型。", "Please select a model.": "请选择一个模型。", @@ -770,7 +770,7 @@ "Positive attitude": "积极的态度", "Prefix ID": "Prefix ID", "Prefix ID is used to avoid conflicts with other connections by adding a prefix to the model IDs - leave empty to disable": "Prefix ID 用于通过为模型 ID 添加前缀来避免与其他连接发生冲突 - 留空则禁用此功能", - "Presence Penalty": "", + "Presence Penalty": "重复惩罚（Presence Penalty）", "Previous 30 days": "过去 30 天", "Previous 7 days": "过去 7 天", "Profile Image": "用户头像", @@ -807,7 +807,7 @@ "Rename": "重命名", "Reorder Models": "重新排序模型", "Repeat Last N": "重复最后 N 次", - "Repeat Penalty (Ollama)": "", + "Repeat Penalty (Ollama)": "重复惩罚（Ollama）", "Reply in Thread": "在主题中回复", "Request Mode": "请求模式", "Reranking Model": "重排模型", @@ -870,7 +870,7 @@ "Select a pipeline": "选择一个管道", "Select a pipeline url": "选择一个管道 URL", "Select a tool": "选择一个工具", - "Select an auth method": "选择身份验证方法", + "Select an auth method": "选择身份验证方式", "Select an Ollama instance": "选择一个 Ollama 实例。", "Select Engine": "选择引擎", "Select Knowledge": "选择知识", @@ -904,10 +904,10 @@ "Set the number of worker threads used for computation. This option controls how many threads are used to process incoming requests concurrently. Increasing this value can improve performance under high concurrency workloads but may also consume more CPU resources.": "设置用于计算的工作线程数量。该选项可控制并发处理传入请求的线程数量。增加该值可以提高高并发工作负载下的性能，但也可能消耗更多的 CPU 资源。", "Set Voice": "设置音色", "Set whisper model": "设置 whisper 模型", - "Sets a flat bias against tokens that have appeared at least once. A higher value (e.g., 1.5) will penalize repetitions more strongly, while a lower value (e.g., 0.9) will be more lenient. At 0, it is disabled. (Default: 0)": "", - "Sets a scaling bias against tokens to penalize repetitions, based on how many times they have appeared. A higher value (e.g., 1.5) will penalize repetitions more strongly, while a lower value (e.g., 0.9) will be more lenient. At 0, it is disabled. (Default: 1.1)": "", + "Sets a flat bias against tokens that have appeared at least once. A higher value (e.g., 1.5) will penalize repetitions more strongly, while a lower value (e.g., 0.9) will be more lenient. At 0, it is disabled. (Default: 0)": "这个设置项用于调整对重复 tokens 的抑制强度。当某个 token 至少出现过一次后，系统会通过 flat bias 参数施加惩罚力度：数值越大（如 1.5），抑制重复的效果越强烈；数值较小（如 0.9）则相对宽容。当设为 0 时，系统会完全关闭这个重复抑制功能（默认值为 0）。", + "Sets a scaling bias against tokens to penalize repetitions, based on how many times they have appeared. A higher value (e.g., 1.5) will penalize repetitions more strongly, while a lower value (e.g., 0.9) will be more lenient. At 0, it is disabled. (Default: 1.1)": "这个参数用于通过 scaling bias 机制抑制重复内容：当某些 tokens 重复出现时，系统会根据它们已出现的次数自动施加惩罚。数值越大（如 1.5）惩罚力度越强，能更有效减少重复；数值较小（如 0.9）则允许更多重复。当设为 0 时完全关闭该功能，默认值设置为 1.1 保持适度抑制。", "Sets how far back for the model to look back to prevent repetition. (Default: 64, 0 = disabled, -1 = num_ctx)": "设置模型回溯多远以防止重复。(默认值：64，0 = 禁用，-1 = num_ctx）", - "Sets the random number seed to use for generation. Setting this to a specific number will make the model generate the same text for the same prompt. (Default: random)": "设置生成文本时使用的随机数种子。将其设置为一个特定的数字将使模型在同一提示下生成相同的文本。 默认值：随机", + "Sets the random number seed to use for generation. Setting this to a specific number will make the model generate the same text for the same prompt. (Default: random)": "设置 random number seed 可以控制模型生成文本的随机起点。如果指定一个具体数字，当输入相同的提示语时，模型每次都会生成完全相同的文本内容（默认是随机选取 seed）。", "Sets the size of the context window used to generate the next token. (Default: 2048)": "设置用于生成下一个 Token 的上下文大小。(默认值：2048）", "Sets the stop sequences to use. When this pattern is encountered, the LLM will stop generating text and return. Multiple stop patterns may be set by specifying multiple separate stop parameters in a modelfile.": "设置要使用的停止序列。遇到这种模式时，大语言模型将停止生成文本并返回。可以通过在模型文件中指定多个单独的停止参数来设置多个停止模式。", "Settings": "设置", @@ -952,7 +952,7 @@ "Tags Generation Prompt": "标签生成提示词", "Tail free sampling is used to reduce the impact of less probable tokens from the output. A higher value (e.g., 2.0) will reduce the impact more, while a value of 1.0 disables this setting. (default: 1)": "Tail free sampling 用于减少输出中可能性较低的标记的影响。数值越大（如 2.0），影响就越小，而数值为 1.0 则会禁用此设置。(默认值：1）", "Tap to interrupt": "点击以中断", - "Tasks": "", + "Tasks": "任务", "Tavily API Key": "Tavily API 密钥", "Tell us more:": "请告诉我们更多细节", "Temperature": "温度 (Temperature)", @@ -975,7 +975,7 @@ "The score should be a value between 0.0 (0%) and 1.0 (100%).": "分值应介于 0.0（0%）和 1.0（100%）之间。", "The temperature of the model. Increasing the temperature will make the model answer more creatively. (Default: 0.8)": "模型的温度。提高温度将使模型更具创造性地回答。（默认值：0.8）", "Theme": "主题", - "Thinking...": "正在思考...", + "Thinking...": "正在深度思考...", "This action cannot be undone. Do you wish to continue?": "此操作无法撤销。是否确认继续？", "This ensures that your valuable conversations are securely saved to your backend database. Thank you!": "这将确保您的宝贵对话被安全地保存到后台数据库中。感谢！", "This is an experimental feature, it may not function as expected and is subject to change at any time.": "这是一个实验功能，可能不会如预期那样工作，而且可能随时发生变化。", @@ -989,8 +989,8 @@ "This will delete all models including custom models and cannot be undone.": "这将删除所有模型，包括自定义模型，且无法撤销。", "This will reset the knowledge base and sync all files. Do you wish to continue?": "这将重置知识库并替换所有文件为目录下文件。确认继续？", "Thorough explanation": "解释较为详细", - "Thought for {{DURATION}}": "已推理 持续 {{DURATION}}", - "Thought for {{DURATION}} seconds": "已推理 持续 {{DURATION}} 秒", + "Thought for {{DURATION}}": "已深度思考 用时 {{DURATION}}", + "Thought for {{DURATION}} seconds": "已深度思考 用时 {{DURATION}} 秒", "Tika": "Tika", "Tika Server URL required.": "请输入 Tika 服务器地址。", "Tiktoken": "Tiktoken", @@ -1011,7 +1011,7 @@ "To select actions here, add them to the \"Functions\" workspace first.": "要在这里选择自动化，请先将其添加到工作空间中的“函数”。", "To select filters here, add them to the \"Functions\" workspace first.": "要在这里选择过滤器，请先将其添加到工作空间中的“函数”。", "To select toolkits here, add them to the \"Tools\" workspace first.": "要在这里选择工具包，请先将其添加到工作空间中的“工具”。", - "Toast notifications for new updates": "新更新的弹窗提示", + "Toast notifications for new updates": "更新后弹窗提示更新内容", "Today": "今天", "Toggle settings": "切换设置", "Toggle sidebar": "切换侧边栏", @@ -1056,7 +1056,7 @@ "Updated": "已更新", "Updated at": "更新于", "Updated At": "更新于", - "Upgrade to a licensed plan for enhanced capabilities, including custom theming and branding, and dedicated support.": "", + "Upgrade to a licensed plan for enhanced capabilities, including custom theming and branding, and dedicated support.": "升级到授权计划以获得增强功能，包括自定义主题与品牌以及专属支持。", "Upload": "上传", "Upload a GGUF model": "上传一个 GGUF 模型", "Upload directory": "上传目录", @@ -1095,7 +1095,7 @@ "Warning:": "警告：", "Warning: Enabling this will allow users to upload arbitrary code on the server.": "警告：启用此功能将允许用户在服务器上上传任意代码。", "Warning: If you update or change your embedding model, you will need to re-import all documents.": "警告：如果您修改了语义向量模型，则需要重新导入所有文档。", - "Warning: Jupyter execution enables arbitrary code execution, posing severe security risks—proceed with extreme caution.": "", + "Warning: Jupyter execution enables arbitrary code execution, posing severe security risks—proceed with extreme caution.": "警告：Jupyter 执行允许任意代码执行，存在严重的安全风险——请极其谨慎地操作。", "Web": "网页", "Web API": "网页 API", "Web Loader Settings": "网页爬取设置", From d4743b1a17ed52405ba205c1c19143b2c23ad15e Mon Sep 17 00:00:00 2001 From: KarlLee830 <61072264+KarlLee830@users.noreply.github.com> Date: Tue, 18 Feb 2025 23:42:22 +0800 Subject: [PATCH 02/37] i18n: Update Chinese Translation --- src/lib/i18n/locales/zh-CN/translation.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/i18n/locales/zh-CN/translation.json b/src/lib/i18n/locales/zh-CN/translation.json index 3b7dfb05d..e891396a3 100644 --- a/src/lib/i18n/locales/zh-CN/translation.json +++ b/src/lib/i18n/locales/zh-CN/translation.json @@ -282,7 +282,7 @@ "Description": "描述", "Didn't fully follow instructions": "没有完全遵照指示", "Direct Connections": "直接连接", - "Direct Connections allow users to connect to their own OpenAI compatible API endpoints.": "直接连接Direct Connections 功能 允许用户连接至其自有的、兼容 OpenAI 的 API 端点。", + "Direct Connections allow users to connect to their own OpenAI compatible API endpoints.": "直接连接功能允许用户连接至其自有的、兼容 OpenAI 的 API 端点。", "Direct Connections settings updated": "直接连接设置已更新", "Disabled": "禁用", "Discover a function": "发现更多函数", From 55b0ac85d187e043ccadd6d31a28cc66f89ad589 Mon Sep 17 00:00:00 2001 From: juxiang <73006913+juquxiang@users.noreply.github.com> Date: Wed, 19 Feb 2025 00:53:38 +0800 Subject: [PATCH 03/37] Added user filtering by email and username Added user filtering by email and username --- src/lib/components/admin/Users/UserList.svelte | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/lib/components/admin/Users/UserList.svelte b/src/lib/components/admin/Users/UserList.svelte index 3f7832517..7f8e516fb 100644 --- a/src/lib/components/admin/Users/UserList.svelte +++ b/src/lib/components/admin/Users/UserList.svelte @@ -85,8 +85,9 @@ return true; } else { let name = user.name.toLowerCase(); + let email = user.email.toLowerCase(); const query = search.toLowerCase(); - return name.includes(query); + return name.includes(query) || email.includes(query); } }) .sort((a, b) => { From 925bfe840b46df424360f230a93e748289df0139 Mon Sep 17 00:00:00 2001 From: mikhail-khludnev Date: Tue, 18 Feb 2025 16:39:02 +0300 Subject: [PATCH 04/37] dedupe results from multiple queries --- backend/open_webui/retrieval/utils.py | 35 ++++++++++++++++----------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/backend/open_webui/retrieval/utils.py b/backend/open_webui/retrieval/utils.py index 437183369..e5ba55878 100644 --- a/backend/open_webui/retrieval/utils.py +++ b/backend/open_webui/retrieval/utils.py @@ -138,37 +138,44 @@ def query_doc_with_hybrid_search( def merge_and_sort_query_results( - query_results: list[dict], k: int, reverse: bool = False + query_results: list[dict], k: int, reverse: bool = False ) -> list[dict]: # Initialize lists to store combined data combined_distances = [] combined_documents = [] combined_metadatas = [] + combined_ids = [] for data in query_results: combined_distances.extend(data["distances"][0]) combined_documents.extend(data["documents"][0]) combined_metadatas.extend(data["metadatas"][0]) + # DISTINCT(chunk_id,file_id) - in case if id (chunk_ids) become ordinals + combined_ids.extend([id + meta["file_id"] for id, meta in zip(data["ids"][0], data["metadatas"][0])]) - # Create a list of tuples (distance, document, metadata) - combined = list(zip(combined_distances, combined_documents, combined_metadatas)) + # Create a list of tuples (distance, document, metadata, ids) + combined = list(zip(combined_distances, combined_documents, combined_metadatas, combined_ids)) # Sort the list based on distances combined.sort(key=lambda x: x[0], reverse=reverse) - # We don't have anything :-( - if not combined: - sorted_distances = [] - sorted_documents = [] - sorted_metadatas = [] - else: + sorted_distances = [] + sorted_documents = [] + sorted_metadatas = [] + # Otherwise we don't have anything :-( + if combined: # Unzip the sorted list - sorted_distances, sorted_documents, sorted_metadatas = zip(*combined) - + all_distances, all_documents, all_metadatas, all_ids = zip(*combined) + seen_ids = set() # Slicing the lists to include only k elements - sorted_distances = list(sorted_distances)[:k] - sorted_documents = list(sorted_documents)[:k] - sorted_metadatas = list(sorted_metadatas)[:k] + for index, id in enumerate(all_ids): + if id not in seen_ids: + sorted_distances.append(all_distances[index]) + sorted_documents.append(all_documents[index]) + sorted_metadatas.append(all_metadatas[index]) + seen_ids.add(id) + if len(sorted_distances) >= k: + break # Create the output dictionary result = { From 6c6be5de886f07c64e170b70865b56718d6809f5 Mon Sep 17 00:00:00 2001 From: Ranjan Mohan Date: Sat, 8 Feb 2025 22:37:24 -0700 Subject: [PATCH 05/37] Fixed an issue with clearing application cookies during OAuth signout Closes #8885. During the OAuth signout flow, although the `token` and `oauth_id_token` cookies were marked for deletion, a new RedirectResponse is created and returned. This does not contain the header info from the he Response object used to mark the cookies to be deleted. Hence the cookies remained. Fixed this by re-using the headers from the other Response object. --- backend/open_webui/routers/auths.py | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/open_webui/routers/auths.py b/backend/open_webui/routers/auths.py index a3f2e8b32..494ba3611 100644 --- a/backend/open_webui/routers/auths.py +++ b/backend/open_webui/routers/auths.py @@ -546,6 +546,7 @@ async def signout(request: Request, response: Response): if logout_url: response.delete_cookie("oauth_id_token") return RedirectResponse( + headers=response.headers, url=f"{logout_url}?id_token_hint={oauth_id_token}" ) else: From 359f8b67f8e9c6c1275fbeac21a4d198adbfd321 Mon Sep 17 00:00:00 2001 From: Timothy Jaeryang Baek Date: Tue, 18 Feb 2025 09:54:31 -0800 Subject: [PATCH 06/37] fix: mobile hover issue --- src/tailwind.css | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/tailwind.css b/src/tailwind.css index e6960f2aa..f4e0c0cdd 100644 --- a/src/tailwind.css +++ b/src/tailwind.css @@ -36,3 +36,5 @@ @apply cursor-pointer; } } + +@custom-variant hover (&:hover); From d0114e0703b21c64419ea121309b1a61f33bcf65 Mon Sep 17 00:00:00 2001 From: Timothy Jaeryang Baek Date: Tue, 18 Feb 2025 09:57:12 -0800 Subject: [PATCH 07/37] fix: temp chat issue --- backend/open_webui/utils/middleware.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/backend/open_webui/utils/middleware.py b/backend/open_webui/utils/middleware.py index 93edc8f72..e09c84f96 100644 --- a/backend/open_webui/utils/middleware.py +++ b/backend/open_webui/utils/middleware.py @@ -1359,7 +1359,15 @@ async def process_chat_response( tool_calls = [] - last_assistant_message = get_last_assistant_message(form_data["messages"]) + last_assistant_message = None + try: + if form_data["messages"][-1]["role"] == "assistant": + last_assistant_message = get_last_assistant_message( + form_data["messages"] + ) + except Exception as e: + pass + content = ( message.get("content", "") if message From fd3c24af4e1ea80a45a4b26271cb8d08fb12a42a Mon Sep 17 00:00:00 2001 From: Chris Pietschmann Date: Tue, 18 Feb 2025 13:25:31 -0500 Subject: [PATCH 08/37] Add AzureStorageProvider --- backend/open_webui/storage/provider.py | 76 +++++++++++++++++++ .../test/apps/webui/storage/test_provider.py | 4 + backend/requirements.txt | 4 + 3 files changed, 84 insertions(+) diff --git a/backend/open_webui/storage/provider.py b/backend/open_webui/storage/provider.py index b03cf0a7e..43f4a6922 100644 --- a/backend/open_webui/storage/provider.py +++ b/backend/open_webui/storage/provider.py @@ -15,12 +15,19 @@ from open_webui.config import ( S3_SECRET_ACCESS_KEY, GCS_BUCKET_NAME, GOOGLE_APPLICATION_CREDENTIALS_JSON, + AZURE_STORAGE_ENDPOINT, + AZURE_STORAGE_CONTAINER_NAME, + AZURE_STORAGE_KEY, STORAGE_PROVIDER, UPLOAD_DIR, ) from google.cloud import storage from google.cloud.exceptions import GoogleCloudError, NotFound from open_webui.constants import ERROR_MESSAGES +from azure.identity import DefaultAzureCredential +from azure.storage.blob import BlobServiceClient +from azure.core.exceptions import ResourceNotFoundError + class StorageProvider(ABC): @@ -221,6 +228,73 @@ class GCSStorageProvider(StorageProvider): LocalStorageProvider.delete_all_files() + +class AzureStorageProvider(StorageProvider): + def __init__(self): + self.endpoint = AZURE_STORAGE_ENDPOINT + self.container_name = AZURE_STORAGE_CONTAINER_NAME + storage_key = AZURE_STORAGE_KEY + + if storage_key: + # Configure using the Azure Storage Account Endpoint and Key + self.blob_service_client = BlobServiceClient( + account_url=self.endpoint, credential=storage_key + ) + else: + # Configure using the Azure Storage Account Endpoint and DefaultAzureCredential + # If the key is not configured, then the DefaultAzureCredential will be used to support Managed Identity authentication + self.blob_service_client = BlobServiceClient( + account_url=self.endpoint, credential=DefaultAzureCredential() + ) + self.container_client = self.blob_service_client.get_container_client(self.container_name) + + def upload_file(self, file: BinaryIO, filename: str) -> Tuple[bytes, str]: + """Handles uploading of the file to Azure Blob Storage.""" + contents, file_path = LocalStorageProvider.upload_file(file, filename) + try: + blob_client = self.container_client.get_blob_client(filename) + blob_client.upload_blob(contents, overwrite=True) + return contents, f"{self.endpoint}/{self.container_name}/{filename}" + except Exception as e: + raise RuntimeError(f"Error uploading file to Azure Blob Storage: {e}") + + def get_file(self, file_path: str) -> str: + """Handles downloading of the file from Azure Blob Storage.""" + try: + filename = file_path.split("/")[-1] + local_file_path = f"{UPLOAD_DIR}/{filename}" + blob_client = self.container_client.get_blob_client(filename) + with open(local_file_path, "wb") as download_file: + download_file.write(blob_client.download_blob().readall()) + return local_file_path + except ResourceNotFoundError as e: + raise RuntimeError(f"Error downloading file from Azure Blob Storage: {e}") + + def delete_file(self, file_path: str) -> None: + """Handles deletion of the file from Azure Blob Storage.""" + try: + filename = file_path.split("/")[-1] + blob_client = self.container_client.get_blob_client(filename) + blob_client.delete_blob() + except ResourceNotFoundError as e: + raise RuntimeError(f"Error deleting file from Azure Blob Storage: {e}") + + # Always delete from local storage + LocalStorageProvider.delete_file(file_path) + + def delete_all_files(self) -> None: + """Handles deletion of all files from Azure Blob Storage.""" + try: + blobs = self.container_client.list_blobs() + for blob in blobs: + self.container_client.delete_blob(blob.name) + except Exception as e: + raise RuntimeError(f"Error deleting all files from Azure Blob Storage: {e}") + + # Always delete from local storage + LocalStorageProvider.delete_all_files() + + def get_storage_provider(storage_provider: str): if storage_provider == "local": Storage = LocalStorageProvider() @@ -228,6 +302,8 @@ def get_storage_provider(storage_provider: str): Storage = S3StorageProvider() elif storage_provider == "gcs": Storage = GCSStorageProvider() + elif storage_provider == "azure": + Storage = AzureStorageProvider() else: raise RuntimeError(f"Unsupported storage provider: {storage_provider}") return Storage diff --git a/backend/open_webui/test/apps/webui/storage/test_provider.py b/backend/open_webui/test/apps/webui/storage/test_provider.py index 863106e75..4c3112526 100644 --- a/backend/open_webui/test/apps/webui/storage/test_provider.py +++ b/backend/open_webui/test/apps/webui/storage/test_provider.py @@ -22,6 +22,7 @@ def test_imports(): provider.LocalStorageProvider provider.S3StorageProvider provider.GCSStorageProvider + provider.AzureStorageProvider provider.Storage @@ -32,6 +33,8 @@ def test_get_storage_provider(): assert isinstance(Storage, provider.S3StorageProvider) Storage = provider.get_storage_provider("gcs") assert isinstance(Storage, provider.GCSStorageProvider) + Storage = provider.get_storage_provider("azure") + assert isinstance(Storage, provider.AzureStorageProvider) with pytest.raises(RuntimeError): provider.get_storage_provider("invalid") @@ -48,6 +51,7 @@ def test_class_instantiation(): provider.LocalStorageProvider() provider.S3StorageProvider() provider.GCSStorageProvider() + provider.AzureStorageProvider() class TestLocalStorageProvider: diff --git a/backend/requirements.txt b/backend/requirements.txt index 9b859b84a..f8e5f6684 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -103,5 +103,9 @@ pytest-docker~=3.1.1 googleapis-common-protos==1.63.2 google-cloud-storage==2.19.0 +azure-identity==1.20.0 +azure-storage-blob==12.24.1 + + ## LDAP ldap3==2.9.1 From e4febfa0974c6433b97f5e483430a33bb88e9941 Mon Sep 17 00:00:00 2001 From: Chris Pietschmann Date: Tue, 18 Feb 2025 13:25:49 -0500 Subject: [PATCH 09/37] Add AzureStorageProvider config options --- backend/open_webui/config.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/backend/open_webui/config.py b/backend/open_webui/config.py index adfdcfec8..6bc2b7636 100644 --- a/backend/open_webui/config.py +++ b/backend/open_webui/config.py @@ -668,6 +668,10 @@ GOOGLE_APPLICATION_CREDENTIALS_JSON = os.environ.get( "GOOGLE_APPLICATION_CREDENTIALS_JSON", None ) +AZURE_STORAGE_ENDPOINT = os.environ.get("AZURE_STORAGE_ENDPOINT", None) +AZURE_STORAGE_CONTAINER_NAME = os.environ.get("AZURE_STORAGE_CONTAINER_NAME", "open-webui") +AZURE_STORAGE_KEY = os.environ.get("AZURE_STORAGE_KEY", None) + #################################### # File Upload DIR #################################### From aee57107bcafac2e7a363c9a9b7b046398c00c55 Mon Sep 17 00:00:00 2001 From: Chris Pietschmann Date: Tue, 18 Feb 2025 13:27:37 -0500 Subject: [PATCH 10/37] Update config.py --- backend/open_webui/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/open_webui/config.py b/backend/open_webui/config.py index 6bc2b7636..176511948 100644 --- a/backend/open_webui/config.py +++ b/backend/open_webui/config.py @@ -669,7 +669,7 @@ GOOGLE_APPLICATION_CREDENTIALS_JSON = os.environ.get( ) AZURE_STORAGE_ENDPOINT = os.environ.get("AZURE_STORAGE_ENDPOINT", None) -AZURE_STORAGE_CONTAINER_NAME = os.environ.get("AZURE_STORAGE_CONTAINER_NAME", "open-webui") +AZURE_STORAGE_CONTAINER_NAME = os.environ.get("AZURE_STORAGE_CONTAINER_NAME", None) AZURE_STORAGE_KEY = os.environ.get("AZURE_STORAGE_KEY", None) #################################### From cc4598c41baea60dbaee2b26eab77d0e489bc876 Mon Sep 17 00:00:00 2001 From: Chris Pietschmann Date: Tue, 18 Feb 2025 13:39:27 -0500 Subject: [PATCH 11/37] Update build-release.yml --- .github/workflows/build-release.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build-release.yml b/.github/workflows/build-release.yml index 443d90419..ac2e28130 100644 --- a/.github/workflows/build-release.yml +++ b/.github/workflows/build-release.yml @@ -4,6 +4,7 @@ on: push: branches: - main # or whatever branch you want to use + - azure-storage jobs: release: From 0b1e30988a2df23ffe89342be8e69ecb9a481a80 Mon Sep 17 00:00:00 2001 From: Elkana Bardugo Date: Tue, 18 Feb 2025 21:08:25 +0200 Subject: [PATCH 12/37] Update MarkdownTokens.svelte More dir="auto" to auto direction on RTL --- .../chat/Messages/Markdown/MarkdownTokens.svelte | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/lib/components/chat/Messages/Markdown/MarkdownTokens.svelte b/src/lib/components/chat/Messages/Markdown/MarkdownTokens.svelte index 36cac4d17..0c5244882 100644 --- a/src/lib/components/chat/Messages/Markdown/MarkdownTokens.svelte +++ b/src/lib/components/chat/Messages/Markdown/MarkdownTokens.svelte @@ -76,7 +76,7 @@ {#if token.type === 'hr'}
{:else if token.type === 'heading'} - + {:else if token.type === 'code'} @@ -176,7 +176,7 @@ {#if token.ordered}
    {#each token.items as item, itemIdx} -
  1. +
  2. {#if item?.task} {#each token.items as item, itemIdx} -
  3. +
  4. {#if item?.task} {/if} {:else if token.type === 'details'} - +
    Date: Tue, 18 Feb 2025 14:09:00 -0500 Subject: [PATCH 13/37] add tests --- .github/workflows/integration-test.disabled | 1 + .../test/apps/webui/storage/test_provider.py | 100 +++++++++++++++++- 2 files changed, 100 insertions(+), 1 deletion(-) diff --git a/.github/workflows/integration-test.disabled b/.github/workflows/integration-test.disabled index b248df4b5..946735c5b 100644 --- a/.github/workflows/integration-test.disabled +++ b/.github/workflows/integration-test.disabled @@ -5,6 +5,7 @@ on: branches: - main - dev + - azure-storage pull_request: branches: - main diff --git a/backend/open_webui/test/apps/webui/storage/test_provider.py b/backend/open_webui/test/apps/webui/storage/test_provider.py index 4c3112526..5cfb1b6a8 100644 --- a/backend/open_webui/test/apps/webui/storage/test_provider.py +++ b/backend/open_webui/test/apps/webui/storage/test_provider.py @@ -3,10 +3,11 @@ import os import boto3 import pytest from botocore.exceptions import ClientError -from moto import mock_aws +from moto import mock_aws, mock_azure from open_webui.storage import provider from gcp_storage_emulator.server import create_server from google.cloud import storage +from azure.storage.blob import BlobServiceClient def mock_upload_dir(monkeypatch, tmp_path): @@ -276,3 +277,100 @@ class TestGCSStorageProvider: assert not (upload_dir / self.filename_extra).exists() assert self.Storage.bucket.get_blob(self.filename) == None assert self.Storage.bucket.get_blob(self.filename_extra) == None + + +class TestAzureStorageProvider: + def __init__(self): + self.Storage = provider.AzureStorageProvider() + self.Storage.container_name = "my-container" + self.file_content = b"test content" + self.filename = "test.txt" + self.filename_extra = "test_exyta.txt" + self.file_bytesio_empty = io.BytesIO() + super().__init__() + + @pytest.fixture(scope="class") + def setup(self, monkeypatch): + connection_string = "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtl6rE4rWlgEoMF1rA==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;" + self.Storage.blob_service_client = BlobServiceClient.from_connection_string(connection_string) + self.Storage.container_client = self.Storage.blob_service_client.get_container_client(self.Storage.container_name) + monkeypatch.setattr(self.Storage, "blob_service_client", self.Storage.blob_service_client) + monkeypatch.setattr(self.Storage, "container_client", self.Storage.container_client) + yield + self.Storage.container_client.delete_container() + + def test_upload_file(self, monkeypatch, tmp_path, setup): + upload_dir = mock_upload_dir(monkeypatch, tmp_path) + # Azure checks + with pytest.raises(Exception): + self.Storage.upload_file(io.BytesIO(self.file_content), self.filename) + self.Storage.create_container() + contents, azure_file_path = self.Storage.upload_file( + io.BytesIO(self.file_content), self.filename + ) + blob = self.Storage.blob_service_client.get_blob_client( + container=self.Storage.container_name, blob=self.filename + ) + assert self.file_content == blob.download_blob().readall() + # local checks + assert (upload_dir / self.filename).exists() + assert (upload_dir / self.filename).read_bytes() == self.file_content + assert contents == self.file_content + assert azure_file_path == "azure://" + self.Storage.container_name + "/" + self.filename + with pytest.raises(ValueError): + self.Storage.upload_file(self.file_bytesio_empty, self.filename) + + def test_get_file(self, monkeypatch, tmp_path, setup): + upload_dir = mock_upload_dir(monkeypatch, tmp_path) + self.Storage.create_container() + contents, azure_file_path = self.Storage.upload_file( + io.BytesIO(self.file_content), self.filename + ) + file_path = self.Storage.get_file(azure_file_path) + assert file_path == str(upload_dir / self.filename) + assert (upload_dir / self.filename).exists() + + def test_delete_file(self, monkeypatch, tmp_path, setup): + upload_dir = mock_upload_dir(monkeypatch, tmp_path) + self.Storage.create_container() + contents, azure_file_path = self.Storage.upload_file( + io.BytesIO(self.file_content), self.filename + ) + assert (upload_dir / self.filename).exists() + self.Storage.delete_file(azure_file_path) + assert not (upload_dir / self.filename).exists() + blob = self.Storage.blob_service_client.get_blob_client( + container=self.Storage.container_name, blob=self.filename + ) + with pytest.raises(Exception): + blob.download_blob().readall() + + def test_delete_all_files(self, monkeypatch, tmp_path, setup): + upload_dir = mock_upload_dir(monkeypatch, tmp_path) + self.Storage.create_container() + self.Storage.upload_file(io.BytesIO(self.file_content), self.filename) + blob = self.Storage.blob_service_client.get_blob_client( + container=self.Storage.container_name, blob=self.filename + ) + assert self.file_content == blob.download_blob().readall() + assert (upload_dir / self.filename).exists() + self.Storage.upload_file(io.BytesIO(self.file_content), self.filename_extra) + blob = self.Storage.blob_service_client.get_blob_client( + container=self.Storage.container_name, blob=self.filename_extra + ) + assert self.file_content == blob.download_blob().readall() + assert (upload_dir / self.filename_extra).exists() + + self.Storage.delete_all_files() + assert not (upload_dir / self.filename).exists() + assert not (upload_dir / self.filename_extra).exists() + blob = self.Storage.blob_service_client.get_blob_client( + container=self.Storage.container_name, blob=self.filename + ) + with pytest.raises(Exception): + blob.download_blob().readall() + blob = self.Storage.blob_service_client.get_blob_client( + container=self.Storage.container_name, blob=self.filename_extra + ) + with pytest.raises(Exception): + blob.download_blob().readall() \ No newline at end of file From cb0666ac216ab5794e4ef795e367929c4dbec91a Mon Sep 17 00:00:00 2001 From: Chris Pietschmann Date: Tue, 18 Feb 2025 14:13:54 -0500 Subject: [PATCH 14/37] update --- .github/workflows/build-release.yml | 1 - .github/workflows/integration-test.disabled | 1 - 2 files changed, 2 deletions(-) diff --git a/.github/workflows/build-release.yml b/.github/workflows/build-release.yml index ac2e28130..443d90419 100644 --- a/.github/workflows/build-release.yml +++ b/.github/workflows/build-release.yml @@ -4,7 +4,6 @@ on: push: branches: - main # or whatever branch you want to use - - azure-storage jobs: release: diff --git a/.github/workflows/integration-test.disabled b/.github/workflows/integration-test.disabled index 946735c5b..b248df4b5 100644 --- a/.github/workflows/integration-test.disabled +++ b/.github/workflows/integration-test.disabled @@ -5,7 +5,6 @@ on: branches: - main - dev - - azure-storage pull_request: branches: - main From 61644f216e5ace539f8de72f691f3ec2c617fb0a Mon Sep 17 00:00:00 2001 From: Elkana Bardugo Date: Tue, 18 Feb 2025 21:30:25 +0200 Subject: [PATCH 15/37] Update CHANGELOG.md --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a61d81f46..358fda15b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.5.15] + +### Added + +- **🌍 Enhanced Internationalization (i18n)**: Improved right-to-left languages experience with automatic text direction handling in chat and sidebar + ## [0.5.14] - 2025-02-17 ### Fixed From b86f8df29f6f2101787350990401b51b0341db22 Mon Sep 17 00:00:00 2001 From: Chris Pietschmann Date: Tue, 18 Feb 2025 14:37:10 -0500 Subject: [PATCH 16/37] Update test_provider.py --- .../test/apps/webui/storage/test_provider.py | 144 +++++++++++------- 1 file changed, 91 insertions(+), 53 deletions(-) diff --git a/backend/open_webui/test/apps/webui/storage/test_provider.py b/backend/open_webui/test/apps/webui/storage/test_provider.py index 5cfb1b6a8..e434523f8 100644 --- a/backend/open_webui/test/apps/webui/storage/test_provider.py +++ b/backend/open_webui/test/apps/webui/storage/test_provider.py @@ -7,7 +7,8 @@ from moto import mock_aws, mock_azure from open_webui.storage import provider from gcp_storage_emulator.server import create_server from google.cloud import storage -from azure.storage.blob import BlobServiceClient +from azure.storage.blob import BlobServiceClient, BlobContainerClient, BlobClient +from unittest.mock import MagicMock def mock_upload_dir(monkeypatch, tmp_path): @@ -279,98 +280,135 @@ class TestGCSStorageProvider: assert self.Storage.bucket.get_blob(self.filename_extra) == None + class TestAzureStorageProvider: def __init__(self): self.Storage = provider.AzureStorageProvider() + self.Storage.endpoint = "https://myaccount.blob.core.windows.net" self.Storage.container_name = "my-container" self.file_content = b"test content" self.filename = "test.txt" - self.filename_extra = "test_exyta.txt" + self.filename_extra = "test_extra.txt" self.file_bytesio_empty = io.BytesIO() super().__init__() - @pytest.fixture(scope="class") + @pytest.fixture def setup(self, monkeypatch): - connection_string = "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtl6rE4rWlgEoMF1rA==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;" - self.Storage.blob_service_client = BlobServiceClient.from_connection_string(connection_string) - self.Storage.container_client = self.Storage.blob_service_client.get_container_client(self.Storage.container_name) - monkeypatch.setattr(self.Storage, "blob_service_client", self.Storage.blob_service_client) - monkeypatch.setattr(self.Storage, "container_client", self.Storage.container_client) + """Mock BlobServiceClient and BlobContainerClient for local testing""" + # Create mock Blob Service Client + mock_blob_service_client = MagicMock() + mock_container_client = MagicMock() + mock_blob_client = MagicMock() + + # Set up return values + mock_blob_service_client.get_container_client.return_value = mock_container_client + mock_container_client.get_blob_client.return_value = mock_blob_client + + # Mock `from_connection_string` and `BlobServiceClient` constructor + monkeypatch.setattr("azure.storage.blob.BlobServiceClient", lambda *_: mock_blob_service_client) + + # Apply to instance variables + self.Storage.blob_service_client = mock_blob_service_client + self.Storage.container_client = mock_container_client + yield - self.Storage.container_client.delete_container() def test_upload_file(self, monkeypatch, tmp_path, setup): + """Test uploading a file to mocked Azure Storage.""" upload_dir = mock_upload_dir(monkeypatch, tmp_path) - # Azure checks + + # Simulate an error when container does not exist + self.Storage.container_client.get_blob_client.side_effect = Exception("Container does not exist") + with pytest.raises(Exception): self.Storage.upload_file(io.BytesIO(self.file_content), self.filename) + + # Reset side effect and create container + self.Storage.container_client.get_blob_client.side_effect = None + self.Storage.create_container() contents, azure_file_path = self.Storage.upload_file( io.BytesIO(self.file_content), self.filename ) - blob = self.Storage.blob_service_client.get_blob_client( - container=self.Storage.container_name, blob=self.filename - ) - assert self.file_content == blob.download_blob().readall() - # local checks + + # Assertions + self.Storage.container_client.get_blob_client.assert_called_with(self.filename) + self.Storage.container_client.get_blob_client().upload_blob.assert_called_once_with(self.file_content, overwrite=True) + + assert contents == self.file_content + assert azure_file_path == f"https://myaccount.blob.core.windows.net/{self.Storage.container_name}/{self.filename}" assert (upload_dir / self.filename).exists() assert (upload_dir / self.filename).read_bytes() == self.file_content - assert contents == self.file_content - assert azure_file_path == "azure://" + self.Storage.container_name + "/" + self.filename + with pytest.raises(ValueError): self.Storage.upload_file(self.file_bytesio_empty, self.filename) def test_get_file(self, monkeypatch, tmp_path, setup): + """Test retrieving a file from mocked Azure Storage.""" upload_dir = mock_upload_dir(monkeypatch, tmp_path) self.Storage.create_container() - contents, azure_file_path = self.Storage.upload_file( - io.BytesIO(self.file_content), self.filename - ) - file_path = self.Storage.get_file(azure_file_path) + + # Mock upload behavior + self.Storage.upload_file(io.BytesIO(self.file_content), self.filename) + + # Mock blob download behavior + self.Storage.container_client.get_blob_client().download_blob().readall.return_value = self.file_content + + file_path = self.Storage.get_file(f"https://myaccount.blob.core.windows.net/{self.Storage.container_name}/{self.filename}") + assert file_path == str(upload_dir / self.filename) assert (upload_dir / self.filename).exists() + assert (upload_dir / self.filename).read_bytes() == self.file_content def test_delete_file(self, monkeypatch, tmp_path, setup): + """Test deleting a file from mocked Azure Storage.""" upload_dir = mock_upload_dir(monkeypatch, tmp_path) self.Storage.create_container() - contents, azure_file_path = self.Storage.upload_file( - io.BytesIO(self.file_content), self.filename - ) - assert (upload_dir / self.filename).exists() - self.Storage.delete_file(azure_file_path) + + # Mock upload + self.Storage.upload_file(io.BytesIO(self.file_content), self.filename) + + # Mock deletion + self.Storage.container_client.get_blob_client().delete_blob.return_value = None + + self.Storage.delete_file(f"https://myaccount.blob.core.windows.net/{self.Storage.container_name}/{self.filename}") + + # Assertions + self.Storage.container_client.get_blob_client().delete_blob.assert_called_once() assert not (upload_dir / self.filename).exists() - blob = self.Storage.blob_service_client.get_blob_client( - container=self.Storage.container_name, blob=self.filename - ) - with pytest.raises(Exception): - blob.download_blob().readall() def test_delete_all_files(self, monkeypatch, tmp_path, setup): + """Test deleting all files from mocked Azure Storage.""" upload_dir = mock_upload_dir(monkeypatch, tmp_path) self.Storage.create_container() - self.Storage.upload_file(io.BytesIO(self.file_content), self.filename) - blob = self.Storage.blob_service_client.get_blob_client( - container=self.Storage.container_name, blob=self.filename - ) - assert self.file_content == blob.download_blob().readall() - assert (upload_dir / self.filename).exists() - self.Storage.upload_file(io.BytesIO(self.file_content), self.filename_extra) - blob = self.Storage.blob_service_client.get_blob_client( - container=self.Storage.container_name, blob=self.filename_extra - ) - assert self.file_content == blob.download_blob().readall() - assert (upload_dir / self.filename_extra).exists() + # Mock file uploads + self.Storage.upload_file(io.BytesIO(self.file_content), self.filename) + self.Storage.upload_file(io.BytesIO(self.file_content), self.filename_extra) + + # Mock listing and deletion behavior + self.Storage.container_client.list_blobs.return_value = [ + {"name": self.filename}, + {"name": self.filename_extra}, + ] + self.Storage.container_client.get_blob_client().delete_blob.return_value = None + + # Call delete all files self.Storage.delete_all_files() + + # Assertions + self.Storage.container_client.list_blobs.assert_called_once() + self.Storage.container_client.get_blob_client().delete_blob.assert_any_call() assert not (upload_dir / self.filename).exists() assert not (upload_dir / self.filename_extra).exists() - blob = self.Storage.blob_service_client.get_blob_client( - container=self.Storage.container_name, blob=self.filename - ) - with pytest.raises(Exception): - blob.download_blob().readall() - blob = self.Storage.blob_service_client.get_blob_client( - container=self.Storage.container_name, blob=self.filename_extra - ) - with pytest.raises(Exception): - blob.download_blob().readall() \ No newline at end of file + + def test_get_file_not_found(self, monkeypatch, setup): + """Test handling when a requested file does not exist.""" + self.Storage.create_container() + + # Mock behavior to raise an error for missing files + self.Storage.container_client.get_blob_client().download_blob.side_effect = Exception("Blob not found") + + with pytest.raises(Exception, match="Blob not found"): + self.Storage.get_file(f"https://myaccount.blob.core.windows.net/{self.Storage.container_name}/{self.filename}") + From 4a9a88b683e0da71dd71a96fc1fb19564f9a053b Mon Sep 17 00:00:00 2001 From: Chris Pietschmann Date: Tue, 18 Feb 2025 14:41:42 -0500 Subject: [PATCH 17/37] Update test_provider.py --- backend/open_webui/test/apps/webui/storage/test_provider.py | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/open_webui/test/apps/webui/storage/test_provider.py b/backend/open_webui/test/apps/webui/storage/test_provider.py index e434523f8..6770add7b 100644 --- a/backend/open_webui/test/apps/webui/storage/test_provider.py +++ b/backend/open_webui/test/apps/webui/storage/test_provider.py @@ -305,6 +305,7 @@ class TestAzureStorageProvider: mock_container_client.get_blob_client.return_value = mock_blob_client # Mock `from_connection_string` and `BlobServiceClient` constructor + monkeypatch.setattr(provider, "BlobServiceClient", lambda *_: mock_blob_service_client) monkeypatch.setattr("azure.storage.blob.BlobServiceClient", lambda *_: mock_blob_service_client) # Apply to instance variables From f674e28263574f33a920c79d282389f0364825d2 Mon Sep 17 00:00:00 2001 From: Chris Pietschmann Date: Tue, 18 Feb 2025 14:45:13 -0500 Subject: [PATCH 18/37] Update test_provider.py --- backend/open_webui/test/apps/webui/storage/test_provider.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/open_webui/test/apps/webui/storage/test_provider.py b/backend/open_webui/test/apps/webui/storage/test_provider.py index 6770add7b..2a8655027 100644 --- a/backend/open_webui/test/apps/webui/storage/test_provider.py +++ b/backend/open_webui/test/apps/webui/storage/test_provider.py @@ -292,7 +292,7 @@ class TestAzureStorageProvider: self.file_bytesio_empty = io.BytesIO() super().__init__() - @pytest.fixture + @pytest.fixture(scope="class") def setup(self, monkeypatch): """Mock BlobServiceClient and BlobContainerClient for local testing""" # Create mock Blob Service Client From 9a2e81f5f01045b424a5b7b30118e2ebbbabc45e Mon Sep 17 00:00:00 2001 From: Chris Pietschmann Date: Tue, 18 Feb 2025 14:49:03 -0500 Subject: [PATCH 19/37] Update test_provider.py --- .../test/apps/webui/storage/test_provider.py | 22 ++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/backend/open_webui/test/apps/webui/storage/test_provider.py b/backend/open_webui/test/apps/webui/storage/test_provider.py index 2a8655027..2ed2c7b0f 100644 --- a/backend/open_webui/test/apps/webui/storage/test_provider.py +++ b/backend/open_webui/test/apps/webui/storage/test_provider.py @@ -283,13 +283,6 @@ class TestGCSStorageProvider: class TestAzureStorageProvider: def __init__(self): - self.Storage = provider.AzureStorageProvider() - self.Storage.endpoint = "https://myaccount.blob.core.windows.net" - self.Storage.container_name = "my-container" - self.file_content = b"test content" - self.filename = "test.txt" - self.filename_extra = "test_extra.txt" - self.file_bytesio_empty = io.BytesIO() super().__init__() @pytest.fixture(scope="class") @@ -305,12 +298,21 @@ class TestAzureStorageProvider: mock_container_client.get_blob_client.return_value = mock_blob_client # Mock `from_connection_string` and `BlobServiceClient` constructor - monkeypatch.setattr(provider, "BlobServiceClient", lambda *_: mock_blob_service_client) monkeypatch.setattr("azure.storage.blob.BlobServiceClient", lambda *_: mock_blob_service_client) + + + self.Storage = provider.AzureStorageProvider() + self.Storage.endpoint = "https://myaccount.blob.core.windows.net" + self.Storage.container_name = "my-container" + self.file_content = b"test content" + self.filename = "test.txt" + self.filename_extra = "test_extra.txt" + self.file_bytesio_empty = io.BytesIO() + # Apply to instance variables - self.Storage.blob_service_client = mock_blob_service_client - self.Storage.container_client = mock_container_client + #self.Storage.blob_service_client = mock_blob_service_client + #self.Storage.container_client = mock_container_client yield From ff5f0c3e3965a5ef44decba7382052548e442a4b Mon Sep 17 00:00:00 2001 From: Chris Pietschmann Date: Tue, 18 Feb 2025 14:51:17 -0500 Subject: [PATCH 20/37] Update test_provider.py --- backend/open_webui/test/apps/webui/storage/test_provider.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/open_webui/test/apps/webui/storage/test_provider.py b/backend/open_webui/test/apps/webui/storage/test_provider.py index 2ed2c7b0f..23d020c13 100644 --- a/backend/open_webui/test/apps/webui/storage/test_provider.py +++ b/backend/open_webui/test/apps/webui/storage/test_provider.py @@ -311,8 +311,8 @@ class TestAzureStorageProvider: self.file_bytesio_empty = io.BytesIO() # Apply to instance variables - #self.Storage.blob_service_client = mock_blob_service_client - #self.Storage.container_client = mock_container_client + self.Storage.blob_service_client = mock_blob_service_client + self.Storage.container_client = mock_container_client yield From 7d1ec2042905513fbd944f8407f6b1bb3ecd8e64 Mon Sep 17 00:00:00 2001 From: Chris Pietschmann Date: Tue, 18 Feb 2025 14:57:51 -0500 Subject: [PATCH 21/37] Update test_provider.py --- backend/open_webui/test/apps/webui/storage/test_provider.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/backend/open_webui/test/apps/webui/storage/test_provider.py b/backend/open_webui/test/apps/webui/storage/test_provider.py index 23d020c13..2bc3ccb64 100644 --- a/backend/open_webui/test/apps/webui/storage/test_provider.py +++ b/backend/open_webui/test/apps/webui/storage/test_provider.py @@ -298,9 +298,9 @@ class TestAzureStorageProvider: mock_container_client.get_blob_client.return_value = mock_blob_client # Mock `from_connection_string` and `BlobServiceClient` constructor - monkeypatch.setattr("azure.storage.blob.BlobServiceClient", lambda *_: mock_blob_service_client) - - + monkeypatch.setattr(azure.storage.blob.BlobServiceClient, lambda *_: mock_blob_service_client) + monkeypatch.setattr(azure.storage.blob.BlobContainerClient, lambda *_: mock_container_client) + monkeypatch.setattr(azure.storage.blob.BlobClient, lambda *_: mock_blob_client) self.Storage = provider.AzureStorageProvider() self.Storage.endpoint = "https://myaccount.blob.core.windows.net" From 56060db29de3287fdf7df124bce2e44f8fef743e Mon Sep 17 00:00:00 2001 From: Chris Pietschmann Date: Tue, 18 Feb 2025 15:01:26 -0500 Subject: [PATCH 22/37] Update test_provider.py --- .../test/apps/webui/storage/test_provider.py | 94 ++++++++----------- 1 file changed, 41 insertions(+), 53 deletions(-) diff --git a/backend/open_webui/test/apps/webui/storage/test_provider.py b/backend/open_webui/test/apps/webui/storage/test_provider.py index 2bc3ccb64..08338375e 100644 --- a/backend/open_webui/test/apps/webui/storage/test_provider.py +++ b/backend/open_webui/test/apps/webui/storage/test_provider.py @@ -282,26 +282,8 @@ class TestGCSStorageProvider: class TestAzureStorageProvider: - def __init__(self): - super().__init__() - - @pytest.fixture(scope="class") - def setup(self, monkeypatch): - """Mock BlobServiceClient and BlobContainerClient for local testing""" - # Create mock Blob Service Client - mock_blob_service_client = MagicMock() - mock_container_client = MagicMock() - mock_blob_client = MagicMock() - - # Set up return values - mock_blob_service_client.get_container_client.return_value = mock_container_client - mock_container_client.get_blob_client.return_value = mock_blob_client - - # Mock `from_connection_string` and `BlobServiceClient` constructor - monkeypatch.setattr(azure.storage.blob.BlobServiceClient, lambda *_: mock_blob_service_client) - monkeypatch.setattr(azure.storage.blob.BlobContainerClient, lambda *_: mock_container_client) - monkeypatch.setattr(azure.storage.blob.BlobClient, lambda *_: mock_blob_client) - + @pytest.fixture(autouse=True) + def setup_storage(self, monkeypatch): self.Storage = provider.AzureStorageProvider() self.Storage.endpoint = "https://myaccount.blob.core.windows.net" self.Storage.container_name = "my-container" @@ -310,34 +292,48 @@ class TestAzureStorageProvider: self.filename_extra = "test_extra.txt" self.file_bytesio_empty = io.BytesIO() - # Apply to instance variables + # Create mock Blob Service Client and related clients + mock_blob_service_client = MagicMock() + mock_container_client = MagicMock() + mock_blob_client = MagicMock() + + # Set up return values for the mock + mock_blob_service_client.get_container_client.return_value = mock_container_client + mock_container_client.get_blob_client.return_value = mock_blob_client + + # Monkeypatch the Azure classes to return our mocks + monkeypatch.setattr( + azure.storage.blob, "BlobServiceClient", lambda *args, **kwargs: mock_blob_service_client + ) + monkeypatch.setattr( + azure.storage.blob, "BlobContainerClient", lambda *args, **kwargs: mock_container_client + ) + monkeypatch.setattr( + azure.storage.blob, "BlobClient", lambda *args, **kwargs: mock_blob_client + ) + + # Apply mocks to the Storage instance self.Storage.blob_service_client = mock_blob_service_client self.Storage.container_client = mock_container_client - yield - - def test_upload_file(self, monkeypatch, tmp_path, setup): - """Test uploading a file to mocked Azure Storage.""" + def test_upload_file(self, monkeypatch, tmp_path): upload_dir = mock_upload_dir(monkeypatch, tmp_path) - + # Simulate an error when container does not exist self.Storage.container_client.get_blob_client.side_effect = Exception("Container does not exist") - with pytest.raises(Exception): self.Storage.upload_file(io.BytesIO(self.file_content), self.filename) # Reset side effect and create container self.Storage.container_client.get_blob_client.side_effect = None - self.Storage.create_container() - contents, azure_file_path = self.Storage.upload_file( - io.BytesIO(self.file_content), self.filename - ) + contents, azure_file_path = self.Storage.upload_file(io.BytesIO(self.file_content), self.filename) # Assertions self.Storage.container_client.get_blob_client.assert_called_with(self.filename) - self.Storage.container_client.get_blob_client().upload_blob.assert_called_once_with(self.file_content, overwrite=True) - + self.Storage.container_client.get_blob_client().upload_blob.assert_called_once_with( + self.file_content, overwrite=True + ) assert contents == self.file_content assert azure_file_path == f"https://myaccount.blob.core.windows.net/{self.Storage.container_name}/{self.filename}" assert (upload_dir / self.filename).exists() @@ -346,42 +342,38 @@ class TestAzureStorageProvider: with pytest.raises(ValueError): self.Storage.upload_file(self.file_bytesio_empty, self.filename) - def test_get_file(self, monkeypatch, tmp_path, setup): - """Test retrieving a file from mocked Azure Storage.""" + def test_get_file(self, monkeypatch, tmp_path): upload_dir = mock_upload_dir(monkeypatch, tmp_path) self.Storage.create_container() # Mock upload behavior self.Storage.upload_file(io.BytesIO(self.file_content), self.filename) - # Mock blob download behavior self.Storage.container_client.get_blob_client().download_blob().readall.return_value = self.file_content - file_path = self.Storage.get_file(f"https://myaccount.blob.core.windows.net/{self.Storage.container_name}/{self.filename}") + file_url = f"https://myaccount.blob.core.windows.net/{self.Storage.container_name}/{self.filename}" + file_path = self.Storage.get_file(file_url) assert file_path == str(upload_dir / self.filename) assert (upload_dir / self.filename).exists() assert (upload_dir / self.filename).read_bytes() == self.file_content - def test_delete_file(self, monkeypatch, tmp_path, setup): - """Test deleting a file from mocked Azure Storage.""" + def test_delete_file(self, monkeypatch, tmp_path): upload_dir = mock_upload_dir(monkeypatch, tmp_path) self.Storage.create_container() - # Mock upload + # Mock file upload self.Storage.upload_file(io.BytesIO(self.file_content), self.filename) - # Mock deletion self.Storage.container_client.get_blob_client().delete_blob.return_value = None - self.Storage.delete_file(f"https://myaccount.blob.core.windows.net/{self.Storage.container_name}/{self.filename}") + file_url = f"https://myaccount.blob.core.windows.net/{self.Storage.container_name}/{self.filename}" + self.Storage.delete_file(file_url) - # Assertions self.Storage.container_client.get_blob_client().delete_blob.assert_called_once() assert not (upload_dir / self.filename).exists() - def test_delete_all_files(self, monkeypatch, tmp_path, setup): - """Test deleting all files from mocked Azure Storage.""" + def test_delete_all_files(self, monkeypatch, tmp_path): upload_dir = mock_upload_dir(monkeypatch, tmp_path) self.Storage.create_container() @@ -396,22 +388,18 @@ class TestAzureStorageProvider: ] self.Storage.container_client.get_blob_client().delete_blob.return_value = None - # Call delete all files self.Storage.delete_all_files() - # Assertions self.Storage.container_client.list_blobs.assert_called_once() self.Storage.container_client.get_blob_client().delete_blob.assert_any_call() assert not (upload_dir / self.filename).exists() assert not (upload_dir / self.filename_extra).exists() - def test_get_file_not_found(self, monkeypatch, setup): - """Test handling when a requested file does not exist.""" + def test_get_file_not_found(self, monkeypatch): self.Storage.create_container() - # Mock behavior to raise an error for missing files + file_url = f"https://myaccount.blob.core.windows.net/{self.Storage.container_name}/{self.filename}" + # Mock behavior to raise an error for missing blobs self.Storage.container_client.get_blob_client().download_blob.side_effect = Exception("Blob not found") - with pytest.raises(Exception, match="Blob not found"): - self.Storage.get_file(f"https://myaccount.blob.core.windows.net/{self.Storage.container_name}/{self.filename}") - + self.Storage.get_file(file_url) \ No newline at end of file From 2c328cc7c9b9b928ed660864208265bad8034538 Mon Sep 17 00:00:00 2001 From: Chris Pietschmann Date: Tue, 18 Feb 2025 15:04:49 -0500 Subject: [PATCH 23/37] Update test_provider.py --- .../test/apps/webui/storage/test_provider.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/backend/open_webui/test/apps/webui/storage/test_provider.py b/backend/open_webui/test/apps/webui/storage/test_provider.py index 08338375e..75a105bc8 100644 --- a/backend/open_webui/test/apps/webui/storage/test_provider.py +++ b/backend/open_webui/test/apps/webui/storage/test_provider.py @@ -284,14 +284,6 @@ class TestGCSStorageProvider: class TestAzureStorageProvider: @pytest.fixture(autouse=True) def setup_storage(self, monkeypatch): - self.Storage = provider.AzureStorageProvider() - self.Storage.endpoint = "https://myaccount.blob.core.windows.net" - self.Storage.container_name = "my-container" - self.file_content = b"test content" - self.filename = "test.txt" - self.filename_extra = "test_extra.txt" - self.file_bytesio_empty = io.BytesIO() - # Create mock Blob Service Client and related clients mock_blob_service_client = MagicMock() mock_container_client = MagicMock() @@ -312,6 +304,15 @@ class TestAzureStorageProvider: azure.storage.blob, "BlobClient", lambda *args, **kwargs: mock_blob_client ) + + self.Storage = provider.AzureStorageProvider() + self.Storage.endpoint = "https://myaccount.blob.core.windows.net" + self.Storage.container_name = "my-container" + self.file_content = b"test content" + self.filename = "test.txt" + self.filename_extra = "test_extra.txt" + self.file_bytesio_empty = io.BytesIO() + # Apply mocks to the Storage instance self.Storage.blob_service_client = mock_blob_service_client self.Storage.container_client = mock_container_client @@ -402,4 +403,4 @@ class TestAzureStorageProvider: # Mock behavior to raise an error for missing blobs self.Storage.container_client.get_blob_client().download_blob.side_effect = Exception("Blob not found") with pytest.raises(Exception, match="Blob not found"): - self.Storage.get_file(file_url) \ No newline at end of file + self.Storage.get_file(file_url) From 9c8c837ab96dbed4b1e6eb3d0135471486179e65 Mon Sep 17 00:00:00 2001 From: Chris Pietschmann Date: Tue, 18 Feb 2025 15:12:35 -0500 Subject: [PATCH 24/37] Update test_provider.py --- .../test/apps/webui/storage/test_provider.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/backend/open_webui/test/apps/webui/storage/test_provider.py b/backend/open_webui/test/apps/webui/storage/test_provider.py index 75a105bc8..3bb71735c 100644 --- a/backend/open_webui/test/apps/webui/storage/test_provider.py +++ b/backend/open_webui/test/apps/webui/storage/test_provider.py @@ -328,7 +328,9 @@ class TestAzureStorageProvider: # Reset side effect and create container self.Storage.container_client.get_blob_client.side_effect = None self.Storage.create_container() - contents, azure_file_path = self.Storage.upload_file(io.BytesIO(self.file_content), self.filename) + contents, azure_file_path = self.Storage.upload_file( + io.BytesIO(self.file_content), self.filename + ) # Assertions self.Storage.container_client.get_blob_client.assert_called_with(self.filename) @@ -336,7 +338,10 @@ class TestAzureStorageProvider: self.file_content, overwrite=True ) assert contents == self.file_content - assert azure_file_path == f"https://myaccount.blob.core.windows.net/{self.Storage.container_name}/{self.filename}" + assert ( + azure_file_path + == f"https://myaccount.blob.core.windows.net/{self.Storage.container_name}/{self.filename}" + ) assert (upload_dir / self.filename).exists() assert (upload_dir / self.filename).read_bytes() == self.file_content @@ -350,7 +355,9 @@ class TestAzureStorageProvider: # Mock upload behavior self.Storage.upload_file(io.BytesIO(self.file_content), self.filename) # Mock blob download behavior - self.Storage.container_client.get_blob_client().download_blob().readall.return_value = self.file_content + self.Storage.container_client.get_blob_client().download_blob().readall.return_value = ( + self.file_content + ) file_url = f"https://myaccount.blob.core.windows.net/{self.Storage.container_name}/{self.filename}" file_path = self.Storage.get_file(file_url) @@ -401,6 +408,8 @@ class TestAzureStorageProvider: file_url = f"https://myaccount.blob.core.windows.net/{self.Storage.container_name}/{self.filename}" # Mock behavior to raise an error for missing blobs - self.Storage.container_client.get_blob_client().download_blob.side_effect = Exception("Blob not found") + self.Storage.container_client.get_blob_client().download_blob.side_effect = ( + Exception("Blob not found") + ) with pytest.raises(Exception, match="Blob not found"): self.Storage.get_file(file_url) From 55bd7a1c6588ec04b76869c4bcbd96183e5059c4 Mon Sep 17 00:00:00 2001 From: Chris Pietschmann Date: Tue, 18 Feb 2025 15:13:30 -0500 Subject: [PATCH 25/37] Update test_provider.py --- .../test/apps/webui/storage/test_provider.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/backend/open_webui/test/apps/webui/storage/test_provider.py b/backend/open_webui/test/apps/webui/storage/test_provider.py index 3bb71735c..77bbcc5e2 100644 --- a/backend/open_webui/test/apps/webui/storage/test_provider.py +++ b/backend/open_webui/test/apps/webui/storage/test_provider.py @@ -290,8 +290,12 @@ class TestAzureStorageProvider: mock_blob_client = MagicMock() # Set up return values for the mock - mock_blob_service_client.get_container_client.return_value = mock_container_client - mock_container_client.get_blob_client.return_value = mock_blob_client + mock_blob_service_client.get_container_client.return_value = ( + mock_container_client + ) + mock_container_client.get_blob_client.return_value = ( + mock_blob_client + ) # Monkeypatch the Azure classes to return our mocks monkeypatch.setattr( @@ -321,7 +325,9 @@ class TestAzureStorageProvider: upload_dir = mock_upload_dir(monkeypatch, tmp_path) # Simulate an error when container does not exist - self.Storage.container_client.get_blob_client.side_effect = Exception("Container does not exist") + self.Storage.container_client.get_blob_client.side_effect = Exception( + "Container does not exist" + ) with pytest.raises(Exception): self.Storage.upload_file(io.BytesIO(self.file_content), self.filename) From 4c352ff9747280f56d03156f4532566a2fa05dd8 Mon Sep 17 00:00:00 2001 From: Chris Pietschmann Date: Tue, 18 Feb 2025 15:14:46 -0500 Subject: [PATCH 26/37] Update test_provider.py --- .../test/apps/webui/storage/test_provider.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/backend/open_webui/test/apps/webui/storage/test_provider.py b/backend/open_webui/test/apps/webui/storage/test_provider.py index 77bbcc5e2..c92dd176b 100644 --- a/backend/open_webui/test/apps/webui/storage/test_provider.py +++ b/backend/open_webui/test/apps/webui/storage/test_provider.py @@ -293,19 +293,23 @@ class TestAzureStorageProvider: mock_blob_service_client.get_container_client.return_value = ( mock_container_client ) - mock_container_client.get_blob_client.return_value = ( - mock_blob_client - ) + mock_container_client.get_blob_client.return_value = mock_blob_client # Monkeypatch the Azure classes to return our mocks monkeypatch.setattr( - azure.storage.blob, "BlobServiceClient", lambda *args, **kwargs: mock_blob_service_client + azure.storage.blob, + "BlobServiceClient", + lambda *args, **kwargs: mock_blob_service_client ) monkeypatch.setattr( - azure.storage.blob, "BlobContainerClient", lambda *args, **kwargs: mock_container_client + azure.storage.blob, + "BlobContainerClient", + lambda *args, **kwargs: mock_container_client ) monkeypatch.setattr( - azure.storage.blob, "BlobClient", lambda *args, **kwargs: mock_blob_client + azure.storage.blob, + "BlobClient", + lambda *args, **kwargs: mock_blob_client ) From a29f83c4e77452416deb28cf6a09632ea0a8606f Mon Sep 17 00:00:00 2001 From: Chris Pietschmann Date: Tue, 18 Feb 2025 15:17:49 -0500 Subject: [PATCH 27/37] updates to formatting --- backend/open_webui/storage/provider.py | 7 ++++--- .../open_webui/test/apps/webui/storage/test_provider.py | 8 +++----- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/backend/open_webui/storage/provider.py b/backend/open_webui/storage/provider.py index 43f4a6922..ae119e39b 100644 --- a/backend/open_webui/storage/provider.py +++ b/backend/open_webui/storage/provider.py @@ -228,7 +228,6 @@ class GCSStorageProvider(StorageProvider): LocalStorageProvider.delete_all_files() - class AzureStorageProvider(StorageProvider): def __init__(self): self.endpoint = AZURE_STORAGE_ENDPOINT @@ -246,7 +245,9 @@ class AzureStorageProvider(StorageProvider): self.blob_service_client = BlobServiceClient( account_url=self.endpoint, credential=DefaultAzureCredential() ) - self.container_client = self.blob_service_client.get_container_client(self.container_name) + self.container_client = self.blob_service_client.get_container_client( + self.container_name + ) def upload_file(self, file: BinaryIO, filename: str) -> Tuple[bytes, str]: """Handles uploading of the file to Azure Blob Storage.""" @@ -293,7 +294,7 @@ class AzureStorageProvider(StorageProvider): # Always delete from local storage LocalStorageProvider.delete_all_files() - + def get_storage_provider(storage_provider: str): if storage_provider == "local": diff --git a/backend/open_webui/test/apps/webui/storage/test_provider.py b/backend/open_webui/test/apps/webui/storage/test_provider.py index c92dd176b..0d5d81f96 100644 --- a/backend/open_webui/test/apps/webui/storage/test_provider.py +++ b/backend/open_webui/test/apps/webui/storage/test_provider.py @@ -299,17 +299,15 @@ class TestAzureStorageProvider: monkeypatch.setattr( azure.storage.blob, "BlobServiceClient", - lambda *args, **kwargs: mock_blob_service_client + lambda *args, **kwargs: mock_blob_service_client, ) monkeypatch.setattr( azure.storage.blob, "BlobContainerClient", - lambda *args, **kwargs: mock_container_client + lambda *args, **kwargs: mock_container_client, ) monkeypatch.setattr( - azure.storage.blob, - "BlobClient", - lambda *args, **kwargs: mock_blob_client + azure.storage.blob, "BlobClient", lambda *args, **kwargs: mock_blob_client ) From 7404494772bb9fee85dfafa2e91e5ee1d8818e6d Mon Sep 17 00:00:00 2001 From: Chris Pietschmann Date: Tue, 18 Feb 2025 15:19:35 -0500 Subject: [PATCH 28/37] formatting --- backend/open_webui/storage/provider.py | 1 - backend/open_webui/test/apps/webui/storage/test_provider.py | 2 -- 2 files changed, 3 deletions(-) diff --git a/backend/open_webui/storage/provider.py b/backend/open_webui/storage/provider.py index ae119e39b..160a45153 100644 --- a/backend/open_webui/storage/provider.py +++ b/backend/open_webui/storage/provider.py @@ -29,7 +29,6 @@ from azure.storage.blob import BlobServiceClient from azure.core.exceptions import ResourceNotFoundError - class StorageProvider(ABC): @abstractmethod def get_file(self, file_path: str) -> str: diff --git a/backend/open_webui/test/apps/webui/storage/test_provider.py b/backend/open_webui/test/apps/webui/storage/test_provider.py index 0d5d81f96..e9b34aa18 100644 --- a/backend/open_webui/test/apps/webui/storage/test_provider.py +++ b/backend/open_webui/test/apps/webui/storage/test_provider.py @@ -280,7 +280,6 @@ class TestGCSStorageProvider: assert self.Storage.bucket.get_blob(self.filename_extra) == None - class TestAzureStorageProvider: @pytest.fixture(autouse=True) def setup_storage(self, monkeypatch): @@ -310,7 +309,6 @@ class TestAzureStorageProvider: azure.storage.blob, "BlobClient", lambda *args, **kwargs: mock_blob_client ) - self.Storage = provider.AzureStorageProvider() self.Storage.endpoint = "https://myaccount.blob.core.windows.net" self.Storage.container_name = "my-container" From 7b5f82ffc783cf3c906954bec1fe18a35a6eb5a2 Mon Sep 17 00:00:00 2001 From: Chris Pietschmann Date: Tue, 18 Feb 2025 15:26:04 -0500 Subject: [PATCH 29/37] Update test_provider.py --- backend/open_webui/test/apps/webui/storage/test_provider.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/open_webui/test/apps/webui/storage/test_provider.py b/backend/open_webui/test/apps/webui/storage/test_provider.py index e9b34aa18..c9c63a008 100644 --- a/backend/open_webui/test/apps/webui/storage/test_provider.py +++ b/backend/open_webui/test/apps/webui/storage/test_provider.py @@ -3,7 +3,7 @@ import os import boto3 import pytest from botocore.exceptions import ClientError -from moto import mock_aws, mock_azure +from moto import mock_aws from open_webui.storage import provider from gcp_storage_emulator.server import create_server from google.cloud import storage From 888ae008679b16d3730ac972a6f0f3102d1f20fe Mon Sep 17 00:00:00 2001 From: Chris Pietschmann Date: Tue, 18 Feb 2025 15:39:47 -0500 Subject: [PATCH 30/37] Update test_provider.py --- backend/open_webui/test/apps/webui/storage/test_provider.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/backend/open_webui/test/apps/webui/storage/test_provider.py b/backend/open_webui/test/apps/webui/storage/test_provider.py index c9c63a008..08e8acb3e 100644 --- a/backend/open_webui/test/apps/webui/storage/test_provider.py +++ b/backend/open_webui/test/apps/webui/storage/test_provider.py @@ -7,7 +7,7 @@ from moto import mock_aws from open_webui.storage import provider from gcp_storage_emulator.server import create_server from google.cloud import storage -from azure.storage.blob import BlobServiceClient, BlobContainerClient, BlobClient +from azure.storage.blob import BlobServiceClient, ContainerClient, BlobClient from unittest.mock import MagicMock @@ -302,7 +302,7 @@ class TestAzureStorageProvider: ) monkeypatch.setattr( azure.storage.blob, - "BlobContainerClient", + "ContainerClient", lambda *args, **kwargs: mock_container_client, ) monkeypatch.setattr( @@ -354,6 +354,8 @@ class TestAzureStorageProvider: with pytest.raises(ValueError): self.Storage.upload_file(self.file_bytesio_empty, self.filename) + assert (true == false).equals(true) + def test_get_file(self, monkeypatch, tmp_path): upload_dir = mock_upload_dir(monkeypatch, tmp_path) self.Storage.create_container() From 9864185b57a0810155e5cbb9ecf3f9fe281dc8b2 Mon Sep 17 00:00:00 2001 From: Chris Pietschmann Date: Tue, 18 Feb 2025 15:49:44 -0500 Subject: [PATCH 31/37] Update test_provider.py --- backend/open_webui/test/apps/webui/storage/test_provider.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/backend/open_webui/test/apps/webui/storage/test_provider.py b/backend/open_webui/test/apps/webui/storage/test_provider.py index 08e8acb3e..accbb05d1 100644 --- a/backend/open_webui/test/apps/webui/storage/test_provider.py +++ b/backend/open_webui/test/apps/webui/storage/test_provider.py @@ -281,7 +281,10 @@ class TestGCSStorageProvider: class TestAzureStorageProvider: - @pytest.fixture(autouse=True) + def __init__(self): + super().__init__() + + @pytest.fixture(scope="class") def setup_storage(self, monkeypatch): # Create mock Blob Service Client and related clients mock_blob_service_client = MagicMock() @@ -354,7 +357,6 @@ class TestAzureStorageProvider: with pytest.raises(ValueError): self.Storage.upload_file(self.file_bytesio_empty, self.filename) - assert (true == false).equals(true) def test_get_file(self, monkeypatch, tmp_path): upload_dir = mock_upload_dir(monkeypatch, tmp_path) From a232f1f34ee5d931fc1d803b526bcdae8b0b9396 Mon Sep 17 00:00:00 2001 From: Chris Pietschmann Date: Tue, 18 Feb 2025 15:53:54 -0500 Subject: [PATCH 32/37] Update test_provider.py --- backend/open_webui/test/apps/webui/storage/test_provider.py | 1 - 1 file changed, 1 deletion(-) diff --git a/backend/open_webui/test/apps/webui/storage/test_provider.py b/backend/open_webui/test/apps/webui/storage/test_provider.py index accbb05d1..a5ef13504 100644 --- a/backend/open_webui/test/apps/webui/storage/test_provider.py +++ b/backend/open_webui/test/apps/webui/storage/test_provider.py @@ -357,7 +357,6 @@ class TestAzureStorageProvider: with pytest.raises(ValueError): self.Storage.upload_file(self.file_bytesio_empty, self.filename) - def test_get_file(self, monkeypatch, tmp_path): upload_dir = mock_upload_dir(monkeypatch, tmp_path) self.Storage.create_container() From e56b5c063ca72fb5fa3f37965730b63e078a42bd Mon Sep 17 00:00:00 2001 From: JoaoCostaIFG Date: Tue, 18 Feb 2025 22:39:32 +0000 Subject: [PATCH 33/37] feat: add Google Imagen/Gemini API image generation Adds support for Gemini API as an image generation backend. By setting the API Base URL to something like 'https://generativelanguage.googleapis.com/v1beta' and providing their API Key, users should be able to start generating images using models like 'imagen-3.0-generate-002'. --- backend/open_webui/config.py | 14 +++++ backend/open_webui/main.py | 5 ++ backend/open_webui/routers/images.py | 63 +++++++++++++++++++ .../components/admin/Settings/Images.svelte | 22 +++++++ 4 files changed, 104 insertions(+) diff --git a/backend/open_webui/config.py b/backend/open_webui/config.py index adfdcfec8..0b147cf12 100644 --- a/backend/open_webui/config.py +++ b/backend/open_webui/config.py @@ -767,6 +767,9 @@ ENABLE_OPENAI_API = PersistentConfig( OPENAI_API_KEY = os.environ.get("OPENAI_API_KEY", "") OPENAI_API_BASE_URL = os.environ.get("OPENAI_API_BASE_URL", "") +GEMINI_API_KEY = os.environ.get("GEMINI_API_KEY", "") +GEMINI_API_BASE_URL = os.environ.get("GEMINI_API_BASE_URL", "") + if OPENAI_API_BASE_URL == "": OPENAI_API_BASE_URL = "https://api.openai.com/v1" @@ -2064,6 +2067,17 @@ IMAGES_OPENAI_API_KEY = PersistentConfig( os.getenv("IMAGES_OPENAI_API_KEY", OPENAI_API_KEY), ) +IMAGES_GEMINI_API_BASE_URL = PersistentConfig( + "IMAGES_GEMINI_API_BASE_URL", + "image_generation.gemini.api_base_url", + os.getenv("IMAGES_GEMINI_API_BASE_URL", GEMINI_API_BASE_URL), +) +IMAGES_GEMINI_API_KEY = PersistentConfig( + "IMAGES_GEMINI_API_KEY", + "image_generation.gemini.api_key", + os.getenv("IMAGES_GEMINI_API_KEY", GEMINI_API_KEY), +) + IMAGE_SIZE = PersistentConfig( "IMAGE_SIZE", "image_generation.size", os.getenv("IMAGE_SIZE", "512x512") ) diff --git a/backend/open_webui/main.py b/backend/open_webui/main.py index a36323151..22b0526da 100644 --- a/backend/open_webui/main.py +++ b/backend/open_webui/main.py @@ -125,6 +125,8 @@ from open_webui.config import ( IMAGE_STEPS, IMAGES_OPENAI_API_BASE_URL, IMAGES_OPENAI_API_KEY, + IMAGES_GEMINI_API_BASE_URL, + IMAGES_GEMINI_API_KEY, # Audio AUDIO_STT_ENGINE, AUDIO_STT_MODEL, @@ -631,6 +633,9 @@ app.state.config.ENABLE_IMAGE_PROMPT_GENERATION = ENABLE_IMAGE_PROMPT_GENERATION app.state.config.IMAGES_OPENAI_API_BASE_URL = IMAGES_OPENAI_API_BASE_URL app.state.config.IMAGES_OPENAI_API_KEY = IMAGES_OPENAI_API_KEY +app.state.config.IMAGES_GEMINI_API_BASE_URL = IMAGES_GEMINI_API_BASE_URL +app.state.config.IMAGES_GEMINI_API_KEY = IMAGES_GEMINI_API_KEY + app.state.config.IMAGE_GENERATION_MODEL = IMAGE_GENERATION_MODEL app.state.config.AUTOMATIC1111_BASE_URL = AUTOMATIC1111_BASE_URL diff --git a/backend/open_webui/routers/images.py b/backend/open_webui/routers/images.py index 4046773de..4c68442b7 100644 --- a/backend/open_webui/routers/images.py +++ b/backend/open_webui/routers/images.py @@ -55,6 +55,10 @@ async def get_config(request: Request, user=Depends(get_admin_user)): "COMFYUI_WORKFLOW": request.app.state.config.COMFYUI_WORKFLOW, "COMFYUI_WORKFLOW_NODES": request.app.state.config.COMFYUI_WORKFLOW_NODES, }, + "gemini": { + "GEMINI_API_BASE_URL": request.app.state.config.IMAGES_GEMINI_API_BASE_URL, + "GEMINI_API_KEY": request.app.state.config.IMAGES_GEMINI_API_KEY, + }, } @@ -78,6 +82,11 @@ class ComfyUIConfigForm(BaseModel): COMFYUI_WORKFLOW_NODES: list[dict] +class GeminiConfigForm(BaseModel): + GEMINI_API_BASE_URL: str + GEMINI_API_KEY: str + + class ConfigForm(BaseModel): enabled: bool engine: str @@ -85,6 +94,7 @@ class ConfigForm(BaseModel): openai: OpenAIConfigForm automatic1111: Automatic1111ConfigForm comfyui: ComfyUIConfigForm + gemini: GeminiConfigForm @router.post("/config/update") @@ -103,6 +113,11 @@ async def update_config( ) request.app.state.config.IMAGES_OPENAI_API_KEY = form_data.openai.OPENAI_API_KEY + request.app.state.config.IMAGES_GEMINI_API_BASE_URL = ( + form_data.gemini.GEMINI_API_BASE_URL + ) + request.app.state.config.IMAGES_GEMINI_API_KEY = form_data.gemini.GEMINI_API_KEY + request.app.state.config.AUTOMATIC1111_BASE_URL = ( form_data.automatic1111.AUTOMATIC1111_BASE_URL ) @@ -155,6 +170,10 @@ async def update_config( "COMFYUI_WORKFLOW": request.app.state.config.COMFYUI_WORKFLOW, "COMFYUI_WORKFLOW_NODES": request.app.state.config.COMFYUI_WORKFLOW_NODES, }, + "gemini": { + "GEMINI_API_BASE_URL": request.app.state.config.IMAGES_GEMINI_API_BASE_URL, + "GEMINI_API_KEY": request.app.state.config.IMAGES_GEMINI_API_KEY, + }, } @@ -224,6 +243,12 @@ def get_image_model(request): if request.app.state.config.IMAGE_GENERATION_MODEL else "dall-e-2" ) + elif request.app.state.config.IMAGE_GENERATION_ENGINE == "gemini": + return ( + request.app.state.config.IMAGE_GENERATION_MODEL + if request.app.state.config.IMAGE_GENERATION_MODEL + else "imagen-3.0-generate-002" + ) elif request.app.state.config.IMAGE_GENERATION_ENGINE == "comfyui": return ( request.app.state.config.IMAGE_GENERATION_MODEL @@ -299,6 +324,10 @@ def get_models(request: Request, user=Depends(get_verified_user)): {"id": "dall-e-2", "name": "DALL·E 2"}, {"id": "dall-e-3", "name": "DALL·E 3"}, ] + elif request.app.state.config.IMAGE_GENERATION_ENGINE == "gemini": + return [ + {"id": "imagen-3-0-generate-002", "name": "imagen-3.0 generate-002"}, + ] elif request.app.state.config.IMAGE_GENERATION_ENGINE == "comfyui": # TODO - get models from comfyui headers = { @@ -483,6 +512,40 @@ async def image_generations( images.append({"url": url}) return images + elif request.app.state.config.IMAGE_GENERATION_ENGINE == "gemini": + headers = {} + headers["Content-Type"] = "application/json" + api_key = request.app.state.config.IMAGES_GEMINI_API_KEY + model = get_image_model(request) + data = { + "instances": {"prompt": form_data.prompt}, + "parameters": { + "sampleCount": form_data.n, + "outputOptions": {"mimeType": "image/png"}, + }, + } + + # Use asyncio.to_thread for the requests.post call + r = await asyncio.to_thread( + requests.post, + url=f"{request.app.state.config.IMAGES_GEMINI_API_BASE_URL}/models/{model}:predict?key={api_key}", + json=data, + headers=headers, + ) + + r.raise_for_status() + res = r.json() + + images = [] + for image in res["predictions"]: + image_data, content_type = load_b64_image_data( + image["bytesBase64Encoded"] + ) + url = upload_image(request, data, image_data, content_type, user) + images.append({"url": url}) + + return images + elif request.app.state.config.IMAGE_GENERATION_ENGINE == "comfyui": data = { "prompt": form_data.prompt, diff --git a/src/lib/components/admin/Settings/Images.svelte b/src/lib/components/admin/Settings/Images.svelte index 4277c4ebd..590886702 100644 --- a/src/lib/components/admin/Settings/Images.svelte +++ b/src/lib/components/admin/Settings/Images.svelte @@ -261,6 +261,9 @@ } else if (config.engine === 'openai' && config.openai.OPENAI_API_KEY === '') { toast.error($i18n.t('OpenAI API Key is required.')); config.enabled = false; + } else if (config.engine === 'gemini' && config.gemini.GEMINI_API_KEY === '') { + toast.error($i18n.t('Gemini API Key is required.')); + config.enabled = false; } } @@ -294,6 +297,7 @@ +
    @@ -605,6 +609,24 @@ /> + {:else if config?.engine === 'gemini'} +
    +
    {$i18n.t('Gemini API Config')}
    + +
    + + + +
    +
    {/if} From 918764a4f7093ec0838ceeff358356e0942978e6 Mon Sep 17 00:00:00 2001 From: JoaoCostaIFG Date: Wed, 19 Feb 2025 00:00:54 +0000 Subject: [PATCH 34/37] fix: Use x-goog-api-key header for Gemini image generation Place the API key in a header instead of a query parameter. This avoids leaking the API key in logs on request failure, etc... --- backend/open_webui/routers/images.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/backend/open_webui/routers/images.py b/backend/open_webui/routers/images.py index 4c68442b7..3288ec6d8 100644 --- a/backend/open_webui/routers/images.py +++ b/backend/open_webui/routers/images.py @@ -515,7 +515,8 @@ async def image_generations( elif request.app.state.config.IMAGE_GENERATION_ENGINE == "gemini": headers = {} headers["Content-Type"] = "application/json" - api_key = request.app.state.config.IMAGES_GEMINI_API_KEY + headers["x-goog-api-key"] = request.app.state.config.IMAGES_GEMINI_API_KEY + model = get_image_model(request) data = { "instances": {"prompt": form_data.prompt}, @@ -528,7 +529,7 @@ async def image_generations( # Use asyncio.to_thread for the requests.post call r = await asyncio.to_thread( requests.post, - url=f"{request.app.state.config.IMAGES_GEMINI_API_BASE_URL}/models/{model}:predict?key={api_key}", + url=f"{request.app.state.config.IMAGES_GEMINI_API_BASE_URL}/models/{model}:predict", json=data, headers=headers, ) From 5639ba423bfaa5340bb7b26e56cb22d341f13be5 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 18 Feb 2025 18:47:56 -0600 Subject: [PATCH 35/37] Fix "Cannot read properties of undefined (reading 'startsWith')" --- src/lib/components/chat/Messages/Markdown/Source.svelte | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/components/chat/Messages/Markdown/Source.svelte b/src/lib/components/chat/Messages/Markdown/Source.svelte index 4eb1fffb7..b7c7513ae 100644 --- a/src/lib/components/chat/Messages/Markdown/Source.svelte +++ b/src/lib/components/chat/Messages/Markdown/Source.svelte @@ -2,7 +2,7 @@ export let token; export let onClick: Function = () => {}; - let attributes: Record = {}; + let attributes: Record = {}; function extractAttributes(input: string): Record { const regex = /(\w+)="([^"]*)"/g; @@ -42,6 +42,6 @@ }} > - {formattedTitle(attributes.title)} + {attributes.title ? formattedTitle(attributes.title) : ''} From 7837843f829941435be03da712b669189ad72ad0 Mon Sep 17 00:00:00 2001 From: = Date: Tue, 18 Feb 2025 21:16:54 -0600 Subject: [PATCH 36/37] Add redirect capability This feature allows the authentication process to redirect to a route passed in the querystring. This allows the /auth route a means of bringing the user to an expected route instead of the main page (root). --- src/routes/auth/+page.svelte | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/routes/auth/+page.svelte b/src/routes/auth/+page.svelte index 60431bcec..02746d26e 100644 --- a/src/routes/auth/+page.svelte +++ b/src/routes/auth/+page.svelte @@ -28,6 +28,12 @@ let ldapUsername = ''; + const querystringValue = (key) => { + const querystring = window.location.search; + const urlParams = new URLSearchParams(querystring); + return urlParams.get(key); + }; + const setSessionUser = async (sessionUser) => { if (sessionUser) { console.log(sessionUser); @@ -39,7 +45,9 @@ $socket.emit('user-join', { auth: { token: sessionUser.token } }); await user.set(sessionUser); await config.set(await getBackendConfig()); - goto('/'); + + const redirectPath = querystringValue('redirect') || '/'; + goto(redirectPath); } }; From 4ef7aff66304f98c871ed88214162c22f1c682f4 Mon Sep 17 00:00:00 2001 From: Timothy Jaeryang Baek Date: Tue, 18 Feb 2025 19:35:22 -0800 Subject: [PATCH 37/37] refac --- backend/open_webui/retrieval/utils.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/backend/open_webui/retrieval/utils.py b/backend/open_webui/retrieval/utils.py index e5ba55878..59490f37f 100644 --- a/backend/open_webui/retrieval/utils.py +++ b/backend/open_webui/retrieval/utils.py @@ -138,7 +138,7 @@ def query_doc_with_hybrid_search( def merge_and_sort_query_results( - query_results: list[dict], k: int, reverse: bool = False + query_results: list[dict], k: int, reverse: bool = False ) -> list[dict]: # Initialize lists to store combined data combined_distances = [] @@ -151,10 +151,17 @@ def merge_and_sort_query_results( combined_documents.extend(data["documents"][0]) combined_metadatas.extend(data["metadatas"][0]) # DISTINCT(chunk_id,file_id) - in case if id (chunk_ids) become ordinals - combined_ids.extend([id + meta["file_id"] for id, meta in zip(data["ids"][0], data["metadatas"][0])]) + combined_ids.extend( + [ + f"{id}-{meta['file_id']}" + for id, meta in zip(data["ids"][0], data["metadatas"][0]) + ] + ) # Create a list of tuples (distance, document, metadata, ids) - combined = list(zip(combined_distances, combined_documents, combined_metadatas, combined_ids)) + combined = list( + zip(combined_distances, combined_documents, combined_metadatas, combined_ids) + ) # Sort the list based on distances combined.sort(key=lambda x: x[0], reverse=reverse)