Skip to content

Commit

Permalink
add capabilities API and image warning (#238103)
Browse files Browse the repository at this point in the history 
* bump distro

* supports vision work

* better promise handling, clean up if statements

* more ui update

* make capabilities optional

* some cleanup

* fix function
  • Loading branch information
@justschen
justschen authored Jan 17, 2025
1 parent f672ef9 commit 4c84c14
Show file tree
Hide file tree
Showing 5 changed files with 74 additions and 41 deletions.
1 change: 1 addition & 0 deletions src/vs/workbench/api/common/extHostLanguageModels.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -180,6 +180,7 @@ export class ExtHostLanguageModels implements ExtHostLanguageModelsShape {
targetExtensions: metadata.extensions,
isDefault: metadata.isDefault,
isUserSelectable: metadata.isUserSelectable,
capabilities: metadata.capabilities,
});

const responseReceivedListener = provider.onDidReceiveLanguageModelResponse2?.(({ extensionId, participant, tokenCount }) => {
Expand Down
50 changes: 32 additions & 18 deletions src/vs/workbench/contrib/chat/browser/chatContentParts/chatAttachmentsContentPart.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import * as dom from '../../../../../base/browser/dom.js';
import { StandardMouseEvent } from '../../../../../base/browser/mouseEvent.js';
import { IManagedHoverTooltipMarkdownString } from '../../../../../base/browser/ui/hover/hover.js';
import { createInstantHoverDelegate } from '../../../../../base/browser/ui/hover/hoverDelegateFactory.js';
import { Promises } from '../../../../../base/common/async.js';
import { Codicon } from '../../../../../base/common/codicons.js';
import { Emitter } from '../../../../../base/common/event.js';
import { Disposable, DisposableStore, IDisposable } from '../../../../../base/common/lifecycle.js';
Expand Down Expand Up @@ -78,6 +79,7 @@ export class ChatAttachmentsContentPart extends Disposable {
this.attachedContextDisposables.clear();
const hoverDelegate = this.attachedContextDisposables.add(createInstantHoverDelegate());

const attachmentInitPromises: Promise<void>[] = [];
this.variables.forEach(async (attachment) => {
let resource = URI.isUri(attachment.value) ? attachment.value : attachment.value && typeof attachment.value === 'object' && 'uri' in attachment.value && URI.isUri(attachment.value.uri) ? attachment.value.uri : undefined;
let range = attachment.value && typeof attachment.value === 'object' && 'range' in attachment.value && Range.isIRange(attachment.value.range) ? attachment.value.range : undefined;
Expand Down Expand Up @@ -129,34 +131,41 @@ export class ChatAttachmentsContentPart extends Disposable {
});
} else if (attachment.isImage) {
ariaLabel = localize('chat.imageAttachment', "Attached image, {0}", attachment.name);

const hoverElement = dom.$('div.chat-attached-context-hover');
hoverElement.setAttribute('aria-label', ariaLabel);

// Custom label
const pillIcon = dom.$('div.chat-attached-context-pill', {}, dom.$('span.codicon.codicon-file-media'));
const pillIcon = dom.$('div.chat-attached-context-pill', {}, dom.$(isAttachmentOmitted ? 'span.codicon.codicon-warning' : 'span.codicon.codicon-file-media'));
const textLabel = dom.$('span.chat-attached-context-custom-text', {}, attachment.name);
widget.appendChild(pillIcon);
widget.appendChild(textLabel);

let buffer: Uint8Array;
try {
if (attachment.value instanceof URI) {
const readFile = await this.fileService.readFile(attachment.value);
buffer = readFile.value.buffer;

} else {
buffer = attachment.value as Uint8Array;
}
await this.createImageElements(buffer, widget, hoverElement);
} catch (error) {
console.error('Error processing attachment:', error);
if (isAttachmentPartialOrOmitted) {
hoverElement.textContent = localize('chat.imageAttachmentHover', "Image was not sent to the model.");
textLabel.style.textDecoration = 'line-through';
this.attachedContextDisposables.add(this.hoverService.setupManagedHover(hoverDelegate, widget, hoverElement, { trapFocus: true }));
} else {
attachmentInitPromises.push(Promises.withAsyncBody(async (resolve) => {
let buffer: Uint8Array;
try {
if (attachment.value instanceof URI) {
const readFile = await this.fileService.readFile(attachment.value);
if (this.attachedContextDisposables.isDisposed) {
return;
}
buffer = readFile.value.buffer;
} else {
buffer = attachment.value as Uint8Array;
}
this.createImageElements(buffer, widget, hoverElement);
} catch (error) {
console.error('Error processing attachment:', error);
}
this.attachedContextDisposables.add(this.hoverService.setupManagedHover(hoverDelegate, widget, hoverElement, { trapFocus: false }));
resolve();
}));
}

widget.style.position = 'relative';
if (!this.attachedContextDisposables.isDisposed) {
this.attachedContextDisposables.add(this.hoverService.setupManagedHover(hoverDelegate, widget, hoverElement));
}
} else if (isPasteVariableEntry(attachment)) {
ariaLabel = localize('chat.attachment', "Attached context, {0}", attachment.name);

Expand Down Expand Up @@ -219,6 +228,11 @@ export class ChatAttachmentsContentPart extends Disposable {
}
}

await Promise.all(attachmentInitPromises);
if (this.attachedContextDisposables.isDisposed) {
return;
}

if (resource) {
widget.style.cursor = 'pointer';
if (!this.attachedContextDisposables.isDisposed) {
Expand Down
58 changes: 35 additions & 23 deletions src/vs/workbench/contrib/chat/browser/chatInputPart.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -442,6 +442,11 @@ export class ChatInputPart extends Disposable implements IHistoryNavigationWidge
}
}

private supportsVision(): boolean {
const model = this.currentLanguageModel ? this.languageModelsService.lookupLanguageModel(this.currentLanguageModel) : undefined;
return model?.capabilities?.vision ?? false;
}

private setCurrentLanguageModelToDefault() {
const defaultLanguageModel = this.languageModelsService.getLanguageModelIds().find(id => this.languageModelsService.lookupLanguageModel(id)?.isDefault);
const hasUserSelectableLanguageModels = this.languageModelsService.getLanguageModelIds().find(id => {
Expand Down Expand Up @@ -792,6 +797,7 @@ export class ChatInputPart extends Disposable implements IHistoryNavigationWidge
onDidChangeModel: this._onDidChangeCurrentLanguageModel.event,
setModel: (modelId: string) => {
this.setCurrentLanguageModelByUser(modelId);
this.renderAttachedContext();
}
};
return this.instantiationService.createInstance(ModelPickerActionViewItem, action, this._currentLanguageModel, itemDelegate, { hoverDelegate: options.hoverDelegate, keybinding: options.keybinding ?? undefined });
Expand Down Expand Up @@ -930,38 +936,44 @@ export class ChatInputPart extends Disposable implements IHistoryNavigationWidge

} else if (attachment.isImage) {
ariaLabel = localize('chat.imageAttachment', "Attached image, {0}", attachment.name);

const hoverElement = dom.$('div.chat-attached-context-hover');
hoverElement.setAttribute('aria-label', ariaLabel);

// Custom label
const pillIcon = dom.$('div.chat-attached-context-pill', {}, dom.$('span.codicon.codicon-file-media'));
const supportsVision = this.supportsVision();
const pillIcon = dom.$('div.chat-attached-context-pill', {}, dom.$(supportsVision ? 'span.codicon.codicon-file-media' : 'span.codicon.codicon-warning'));
const textLabel = dom.$('span.chat-attached-context-custom-text', {}, attachment.name);
widget.appendChild(pillIcon);
widget.appendChild(textLabel);

attachmentInitPromises.push(Promises.withAsyncBody(async (resolve) => {
let buffer: Uint8Array;
try {
this.attachButtonAndDisposables(widget, index, attachment, hoverDelegate);
if (attachment.value instanceof URI) {
const readFile = await this.fileService.readFile(attachment.value);
if (store.isDisposed) {
return;
if (!supportsVision) {
widget.classList.add('warning');
hoverElement.textContent = localize('chat.imageAttachmentHover', "{0} does not support images.", this.currentLanguageModel ? this.languageModelsService.lookupLanguageModel(this.currentLanguageModel)?.name : this.currentLanguageModel);
textLabel.style.textDecoration = 'line-through';
store.add(this.hoverService.setupManagedHover(hoverDelegate, widget, hoverElement, { trapFocus: true }));
this.attachButtonAndDisposables(widget, index, attachment, hoverDelegate);
} else {
attachmentInitPromises.push(Promises.withAsyncBody(async (resolve) => {
let buffer: Uint8Array;
try {
this.attachButtonAndDisposables(widget, index, attachment, hoverDelegate);
if (attachment.value instanceof URI) {
const readFile = await this.fileService.readFile(attachment.value);
if (store.isDisposed) {
return;
}
buffer = readFile.value.buffer;
} else {
buffer = attachment.value as Uint8Array;
}
buffer = readFile.value.buffer;
} else {
buffer = attachment.value as Uint8Array;
this.createImageElements(buffer, widget, hoverElement);
} catch (error) {
console.error('Error processing attachment:', error);
}
this.createImageElements(buffer, widget, hoverElement);
} catch (error) {
console.error('Error processing attachment:', error);
}

widget.style.position = 'relative';
store.add(this.hoverService.setupManagedHover(hoverDelegate, widget, hoverElement, { trapFocus: false }));
resolve();
}));
store.add(this.hoverService.setupManagedHover(hoverDelegate, widget, hoverElement, { trapFocus: false }));
resolve();
}));
}
widget.style.position = 'relative';
} else if (isPasteVariableEntry(attachment)) {
ariaLabel = localize('chat.attachment', "Attached context, {0}", attachment.name);

Expand Down
3 changes: 3 additions & 0 deletions src/vs/workbench/contrib/chat/common/languageModels.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,9 @@ export interface ILanguageModelChatMetadata {
readonly providerLabel: string;
readonly accountLabel?: string;
};
readonly capabilities?: {
readonly vision?: boolean;
};
}

export interface ILanguageModelChatResponse {
Expand Down
3 changes: 3 additions & 0 deletions src/vscode-dts/vscode.proposed.chatProvider.d.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,9 @@ declare module 'vscode' {
// TODO@API maybe an enum, LanguageModelChatProviderPickerAvailability?
readonly isDefault?: boolean;
readonly isUserSelectable?: boolean;
readonly capabilities?: {
readonly vision?: boolean;
};
}

export interface ChatResponseProviderMetadata {
Expand Down

0 comments on commit 4c84c14

Please sign in to comment.