Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

add capabilities API and image warning #238103

Merged
merged 8 commits into from
Jan 17, 2025
Merged

add capabilities API and image warning #238103

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
57b6c1a
bump distro
justschen Jan 14, 2025
52ef1df
supports vision work
justschen Jan 16, 2025
5ec9ce4
better promise handling, clean up if statements
justschen Jan 17, 2025
62cd31e
Merge branch 'main' into justin/geodude
justschen Jan 17, 2025
2924a91
more ui update
justschen Jan 17, 2025
966f0a9
make capabilities optional
justschen Jan 17, 2025
67f9df5
some cleanup
justschen Jan 17, 2025
3689da7
fix function
justschen Jan 17, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions src/vs/workbench/api/common/extHostLanguageModels.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -180,6 +180,7 @@ export class ExtHostLanguageModels implements ExtHostLanguageModelsShape {
targetExtensions: metadata.extensions,
isDefault: metadata.isDefault,
isUserSelectable: metadata.isUserSelectable,
capabilities: metadata.capabilities,
});

const responseReceivedListener = provider.onDidReceiveLanguageModelResponse2?.(({ extensionId, participant, tokenCount }) => {
Expand Down
50 changes: 32 additions & 18 deletions src/vs/workbench/contrib/chat/browser/chatContentParts/chatAttachmentsContentPart.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import * as dom from '../../../../../base/browser/dom.js';
import { StandardMouseEvent } from '../../../../../base/browser/mouseEvent.js';
import { IManagedHoverTooltipMarkdownString } from '../../../../../base/browser/ui/hover/hover.js';
import { createInstantHoverDelegate } from '../../../../../base/browser/ui/hover/hoverDelegateFactory.js';
import { Promises } from '../../../../../base/common/async.js';
import { Codicon } from '../../../../../base/common/codicons.js';
import { Emitter } from '../../../../../base/common/event.js';
import { Disposable, DisposableStore, IDisposable } from '../../../../../base/common/lifecycle.js';
Expand Down Expand Up @@ -78,6 +79,7 @@ export class ChatAttachmentsContentPart extends Disposable {
this.attachedContextDisposables.clear();
const hoverDelegate = this.attachedContextDisposables.add(createInstantHoverDelegate());

const attachmentInitPromises: Promise<void>[] = [];
this.variables.forEach(async (attachment) => {
let resource = URI.isUri(attachment.value) ? attachment.value : attachment.value && typeof attachment.value === 'object' && 'uri' in attachment.value && URI.isUri(attachment.value.uri) ? attachment.value.uri : undefined;
let range = attachment.value && typeof attachment.value === 'object' && 'range' in attachment.value && Range.isIRange(attachment.value.range) ? attachment.value.range : undefined;
Expand Down Expand Up @@ -129,34 +131,41 @@ export class ChatAttachmentsContentPart extends Disposable {
});
} else if (attachment.isImage) {
ariaLabel = localize('chat.imageAttachment', "Attached image, {0}", attachment.name);

const hoverElement = dom.$('div.chat-attached-context-hover');
hoverElement.setAttribute('aria-label', ariaLabel);

// Custom label
const pillIcon = dom.$('div.chat-attached-context-pill', {}, dom.$('span.codicon.codicon-file-media'));
const pillIcon = dom.$('div.chat-attached-context-pill', {}, dom.$(isAttachmentOmitted ? 'span.codicon.codicon-warning' : 'span.codicon.codicon-file-media'));
const textLabel = dom.$('span.chat-attached-context-custom-text', {}, attachment.name);
widget.appendChild(pillIcon);
widget.appendChild(textLabel);

let buffer: Uint8Array;
try {
if (attachment.value instanceof URI) {
const readFile = await this.fileService.readFile(attachment.value);
buffer = readFile.value.buffer;

} else {
buffer = attachment.value as Uint8Array;
}
await this.createImageElements(buffer, widget, hoverElement);
} catch (error) {
console.error('Error processing attachment:', error);
if (isAttachmentPartialOrOmitted) {
hoverElement.textContent = localize('chat.imageAttachmentHover', "Image was not sent to the model.");
textLabel.style.textDecoration = 'line-through';
this.attachedContextDisposables.add(this.hoverService.setupManagedHover(hoverDelegate, widget, hoverElement, { trapFocus: true }));
} else {
attachmentInitPromises.push(Promises.withAsyncBody(async (resolve) => {
let buffer: Uint8Array;
try {
if (attachment.value instanceof URI) {
const readFile = await this.fileService.readFile(attachment.value);
if (this.attachedContextDisposables.isDisposed) {
return;
}
buffer = readFile.value.buffer;
} else {
buffer = attachment.value as Uint8Array;
}
this.createImageElements(buffer, widget, hoverElement);
} catch (error) {
console.error('Error processing attachment:', error);
}
this.attachedContextDisposables.add(this.hoverService.setupManagedHover(hoverDelegate, widget, hoverElement, { trapFocus: false }));
resolve();
}));
}

widget.style.position = 'relative';
if (!this.attachedContextDisposables.isDisposed) {
this.attachedContextDisposables.add(this.hoverService.setupManagedHover(hoverDelegate, widget, hoverElement));
}
} else if (isPasteVariableEntry(attachment)) {
ariaLabel = localize('chat.attachment', "Attached context, {0}", attachment.name);

Expand Down Expand Up @@ -219,6 +228,11 @@ export class ChatAttachmentsContentPart extends Disposable {
}
}

await Promise.all(attachmentInitPromises);
if (this.attachedContextDisposables.isDisposed) {
return;
}

if (resource) {
widget.style.cursor = 'pointer';
if (!this.attachedContextDisposables.isDisposed) {
Expand Down
58 changes: 35 additions & 23 deletions src/vs/workbench/contrib/chat/browser/chatInputPart.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -444,6 +444,11 @@ export class ChatInputPart extends Disposable implements IHistoryNavigationWidge
}
}

private supportsVision(): boolean {
const model = this.currentLanguageModel ? this.languageModelsService.lookupLanguageModel(this.currentLanguageModel) : undefined;
return model?.capabilities?.vision ?? false;
}

private setCurrentLanguageModelToDefault() {
const defaultLanguageModel = this.languageModelsService.getLanguageModelIds().find(id => this.languageModelsService.lookupLanguageModel(id)?.isDefault);
const hasUserSelectableLanguageModels = this.languageModelsService.getLanguageModelIds().find(id => {
Expand Down Expand Up @@ -796,6 +801,7 @@ export class ChatInputPart extends Disposable implements IHistoryNavigationWidge
onDidChangeModel: this._onDidChangeCurrentLanguageModel.event,
setModel: (modelId: string) => {
this.setCurrentLanguageModelByUser(modelId);
this.renderAttachedContext();
}
};
return this.instantiationService.createInstance(ModelPickerActionViewItem, action, this._currentLanguageModel, itemDelegate, { hoverDelegate: options.hoverDelegate, keybinding: options.keybinding ?? undefined });
Expand Down Expand Up @@ -934,38 +940,44 @@ export class ChatInputPart extends Disposable implements IHistoryNavigationWidge

} else if (attachment.isImage) {
ariaLabel = localize('chat.imageAttachment', "Attached image, {0}", attachment.name);

const hoverElement = dom.$('div.chat-attached-context-hover');
hoverElement.setAttribute('aria-label', ariaLabel);

// Custom label
const pillIcon = dom.$('div.chat-attached-context-pill', {}, dom.$('span.codicon.codicon-file-media'));
const supportsVision = this.supportsVision();
const pillIcon = dom.$('div.chat-attached-context-pill', {}, dom.$(supportsVision ? 'span.codicon.codicon-file-media' : 'span.codicon.codicon-warning'));
const textLabel = dom.$('span.chat-attached-context-custom-text', {}, attachment.name);
widget.appendChild(pillIcon);
widget.appendChild(textLabel);

attachmentInitPromises.push(Promises.withAsyncBody(async (resolve) => {
let buffer: Uint8Array;
try {
this.attachButtonAndDisposables(widget, index, attachment, hoverDelegate);
if (attachment.value instanceof URI) {
const readFile = await this.fileService.readFile(attachment.value);
if (store.isDisposed) {
return;
if (!supportsVision) {
widget.classList.add('warning');
hoverElement.textContent = localize('chat.imageAttachmentHover', "{0} does not support images.", this.currentLanguageModel ? this.languageModelsService.lookupLanguageModel(this.currentLanguageModel)?.name : this.currentLanguageModel);
textLabel.style.textDecoration = 'line-through';
store.add(this.hoverService.setupManagedHover(hoverDelegate, widget, hoverElement, { trapFocus: true }));
this.attachButtonAndDisposables(widget, index, attachment, hoverDelegate);
} else {
attachmentInitPromises.push(Promises.withAsyncBody(async (resolve) => {
let buffer: Uint8Array;
try {
this.attachButtonAndDisposables(widget, index, attachment, hoverDelegate);
if (attachment.value instanceof URI) {
const readFile = await this.fileService.readFile(attachment.value);
if (store.isDisposed) {
return;
}
buffer = readFile.value.buffer;
} else {
buffer = attachment.value as Uint8Array;
}
buffer = readFile.value.buffer;
} else {
buffer = attachment.value as Uint8Array;
this.createImageElements(buffer, widget, hoverElement);
} catch (error) {
console.error('Error processing attachment:', error);
}
this.createImageElements(buffer, widget, hoverElement);
} catch (error) {
console.error('Error processing attachment:', error);
}

widget.style.position = 'relative';
store.add(this.hoverService.setupManagedHover(hoverDelegate, widget, hoverElement, { trapFocus: false }));
resolve();
}));
store.add(this.hoverService.setupManagedHover(hoverDelegate, widget, hoverElement, { trapFocus: false }));
resolve();
}));
}
widget.style.position = 'relative';
} else if (isPasteVariableEntry(attachment)) {
ariaLabel = localize('chat.attachment', "Attached context, {0}", attachment.name);

Expand Down
3 changes: 3 additions & 0 deletions src/vs/workbench/contrib/chat/common/languageModels.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,9 @@ export interface ILanguageModelChatMetadata {
readonly providerLabel: string;
readonly accountLabel?: string;
};
readonly capabilities?: {
readonly vision?: boolean;
};
}

export interface ILanguageModelChatResponse {
Expand Down
3 changes: 3 additions & 0 deletions src/vscode-dts/vscode.proposed.chatProvider.d.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,9 @@ declare module 'vscode' {
// TODO@API maybe an enum, LanguageModelChatProviderPickerAvailability?
readonly isDefault?: boolean;
readonly isUserSelectable?: boolean;
readonly capabilities?: {
readonly vision?: boolean;
};
}

export interface ChatResponseProviderMetadata {
Expand Down
Loading